You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by pn...@apache.org on 2014/09/28 10:50:05 UTC
[01/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Repository: lucenenet
Updated Branches:
refs/heads/master 3b226e8bf -> d852d5b04
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs b/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs
index b4ba4cf..605f7bd 100644
--- a/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs
+++ b/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,184 +15,209 @@ package codecs.sep;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Arrays;
-
-import store.IndexOutput;
-import codecs.MultiLevelSkipListWriter;
-import index.FieldInfo.IndexOptions;
-
-// TODO: -- skip data should somehow be more local to the
-// particular stream (doc, freq, pos, payload)
-
-/**
- * Implements the skip list writer for the default posting list format
- * that stores positions and payloads.
- *
- * @lucene.experimental
- */
-class SepSkipListWriter extends MultiLevelSkipListWriter {
- private int[] lastSkipDoc;
- private int[] lastSkipPayloadLength;
- private long[] lastSkipPayloadPointer;
-
- private IntIndexOutput.Index[] docIndex;
- private IntIndexOutput.Index[] freqIndex;
- private IntIndexOutput.Index[] posIndex;
-
- private IntIndexOutput freqOutput;
- // TODO: -- private again
- IntIndexOutput posOutput;
- // TODO: -- private again
- IndexOutput payloadOutput;
-
- private int curDoc;
- private bool curStorePayloads;
- private int curPayloadLength;
- private long curPayloadPointer;
-
- SepSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount,
- IntIndexOutput freqOutput,
- IntIndexOutput docOutput,
- IntIndexOutput posOutput,
- IndexOutput payloadOutput)
- {
- super(skipInterval, numberOfSkipLevels, docCount);
-
- this.freqOutput = freqOutput;
- this.posOutput = posOutput;
- this.payloadOutput = payloadOutput;
-
- lastSkipDoc = new int[numberOfSkipLevels];
- lastSkipPayloadLength = new int[numberOfSkipLevels];
- // TODO: -- also cutover normal IndexOutput to use getIndex()?
- lastSkipPayloadPointer = new long[numberOfSkipLevels];
-
- freqIndex = new IntIndexOutput.Index[numberOfSkipLevels];
- docIndex = new IntIndexOutput.Index[numberOfSkipLevels];
- posIndex = new IntIndexOutput.Index[numberOfSkipLevels];
-
- for(int i=0;i<numberOfSkipLevels;i++) {
- if (freqOutput != null) {
- freqIndex[i] = freqOutput.index();
- }
- docIndex[i] = docOutput.index();
- if (posOutput != null) {
- posIndex[i] = posOutput.index();
- }
- }
- }
-
- IndexOptions indexOptions;
-
- void setIndexOptions(IndexOptions v) {
- indexOptions = v;
- }
-
- void setPosOutput(IntIndexOutput posOutput) {
- this.posOutput = posOutput;
- for(int i=0;i<numberOfSkipLevels;i++) {
- posIndex[i] = posOutput.index();
- }
- }
-
- void setPayloadOutput(IndexOutput payloadOutput) {
- this.payloadOutput = payloadOutput;
- }
-
- /**
- * Sets the values for the current skip data.
- */
- // Called @ every index interval (every 128th (by default)
- // doc)
- void setSkipData(int doc, bool storePayloads, int payloadLength) {
- this.curDoc = doc;
- this.curStorePayloads = storePayloads;
- this.curPayloadLength = payloadLength;
- if (payloadOutput != null) {
- this.curPayloadPointer = payloadOutput.getFilePointer();
- }
- }
-
- // Called @ start of new term
- protected void resetSkip(IntIndexOutput.Index topDocIndex, IntIndexOutput.Index topFreqIndex, IntIndexOutput.Index topPosIndex)
- {
- super.resetSkip();
-
- Arrays.fill(lastSkipDoc, 0);
- Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list
- for(int i=0;i<numberOfSkipLevels;i++) {
- docIndex[i].copyFrom(topDocIndex, true);
- if (freqOutput != null) {
- freqIndex[i].copyFrom(topFreqIndex, true);
- }
- if (posOutput != null) {
- posIndex[i].copyFrom(topPosIndex, true);
- }
- }
- if (payloadOutput != null) {
- Arrays.fill(lastSkipPayloadPointer, payloadOutput.getFilePointer());
- }
- }
-
- @Override
- protected void writeSkipData(int level, IndexOutput skipBuffer) {
- // To efficiently store payloads in the posting lists we do not store the length of
- // every payload. Instead we omit the length for a payload if the previous payload had
- // the same length.
- // However, in order to support skipping the payload length at every skip point must be known.
- // So we use the same length encoding that we use for the posting lists for the skip data as well:
- // Case 1: current field does not store payloads
- // SkipDatum --> DocSkip, FreqSkip, ProxSkip
- // DocSkip,FreqSkip,ProxSkip --> VInt
- // DocSkip records the document number before every SkipInterval th document in TermFreqs.
- // Document numbers are represented as differences from the previous value in the sequence.
- // Case 2: current field stores payloads
- // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip
- // DocSkip,FreqSkip,ProxSkip --> VInt
- // PayloadLength --> VInt
- // In this case DocSkip/2 is the difference between
- // the current and the previous value. If DocSkip
- // is odd, then a PayloadLength encoded as VInt follows,
- // if DocSkip is even, then it is assumed that the
- // current payload length equals the length at the previous
- // skip point
-
- Debug.Assert( indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !curStorePayloads;
-
- if (curStorePayloads) {
- int delta = curDoc - lastSkipDoc[level];
- if (curPayloadLength == lastSkipPayloadLength[level]) {
- // the current payload length equals the length at the previous skip point,
- // so we don't store the length again
- skipBuffer.writeVInt(delta << 1);
- } else {
- // the payload length is different from the previous one. We shift the DocSkip,
- // set the lowest bit and store the current payload length as VInt.
- skipBuffer.writeVInt(delta << 1 | 1);
- skipBuffer.writeVInt(curPayloadLength);
- lastSkipPayloadLength[level] = curPayloadLength;
- }
- } else {
- // current field does not store payloads
- skipBuffer.writeVInt(curDoc - lastSkipDoc[level]);
- }
-
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- freqIndex[level].mark();
- freqIndex[level].write(skipBuffer, false);
- }
- docIndex[level].mark();
- docIndex[level].write(skipBuffer, false);
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- posIndex[level].mark();
- posIndex[level].write(skipBuffer, false);
- if (curStorePayloads) {
- skipBuffer.writeVInt((int) (curPayloadPointer - lastSkipPayloadPointer[level]));
- }
- }
-
- lastSkipDoc[level] = curDoc;
- lastSkipPayloadPointer[level] = curPayloadPointer;
- }
-}
+namespace Lucene.Net.Codecs.Sep
+{
+ using System.Diagnostics;
+ using Index;
+ using Store;
+ using Support;
+
+ /// <summary>
+ /// Implements the skip list writer for the default posting list format
+ /// that stores positions and payloads.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ /// <remarks>
+ /// TODO: -- skip data should somehow be more local to the particular stream
+ /// (doc, freq, pos, payload)
+ /// </remarks>
+ internal class SepSkipListWriter : MultiLevelSkipListWriter
+ {
+ private readonly int[] _lastSkipDoc;
+ private readonly int[] _lastSkipPayloadLength;
+ private readonly long[] _lastSkipPayloadPointer;
+ private FieldInfo.IndexOptions _indexOptions;
+
+ private readonly IntIndexOutputIndex[] _docIndex;
+ private readonly IntIndexOutputIndex[] _freqIndex;
+ private readonly IntIndexOutputIndex[] _posIndex;
+
+ private readonly IntIndexOutput _freqOutput;
+ private IntIndexOutput _posOutput;
+ private IndexOutput _payloadOutput;
+
+ private int _curDoc;
+ private bool _curStorePayloads;
+ private int _curPayloadLength;
+ private long _curPayloadPointer;
+
+ internal SepSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IntIndexOutput freqOutput,
+ IntIndexOutput docOutput, IntIndexOutput posOutput, IndexOutput payloadOutput)
+ : base(skipInterval, numberOfSkipLevels, docCount)
+ {
+
+ _freqOutput = freqOutput;
+ _posOutput = posOutput;
+ _payloadOutput = payloadOutput;
+
+ _lastSkipDoc = new int[numberOfSkipLevels];
+ _lastSkipPayloadLength = new int[numberOfSkipLevels];
+ // TODO: -- also cutover normal IndexOutput to use getIndex()?
+ _lastSkipPayloadPointer = new long[numberOfSkipLevels];
+
+ _freqIndex = new IntIndexOutputIndex[numberOfSkipLevels];
+ _docIndex = new IntIndexOutputIndex[numberOfSkipLevels];
+ _posIndex = new IntIndexOutputIndex[numberOfSkipLevels];
+
+ for (var i = 0; i < numberOfSkipLevels; i++)
+ {
+ if (freqOutput != null)
+ {
+ _freqIndex[i] = freqOutput.Index();
+ }
+ _docIndex[i] = docOutput.Index();
+ if (posOutput != null)
+ {
+ _posIndex[i] = posOutput.Index();
+ }
+ }
+ }
+
+ internal virtual FieldInfo.IndexOptions IndexOptions
+ {
+ set { _indexOptions = value; }
+ }
+
+ internal virtual IntIndexOutput PosOutput
+ {
+ set
+ {
+ _posOutput = value;
+ for (var i = 0; i < NumberOfSkipLevels; i++)
+ {
+ _posIndex[i] = value.Index();
+ }
+ }
+ }
+
+ internal virtual IndexOutput PayloadOutput
+ {
+ set { _payloadOutput = value; }
+ }
+
+ /// <summary>
+ /// Sets the values for the current skip data.
+ /// Called @ every index interval (every 128th (by default) doc)
+ /// </summary>
+ internal virtual void SetSkipData(int doc, bool storePayloads, int payloadLength)
+ {
+ _curDoc = doc;
+ _curStorePayloads = storePayloads;
+ _curPayloadLength = payloadLength;
+ if (_payloadOutput != null)
+ {
+ _curPayloadPointer = _payloadOutput.FilePointer;
+ }
+ }
+
+ /// <summary>
+ /// Called @ start of new term
+ /// </summary>
+ protected internal virtual void ResetSkip(IntIndexOutputIndex topDocIndex, IntIndexOutputIndex topFreqIndex,
+ IntIndexOutputIndex topPosIndex)
+ {
+ base.ResetSkip();
+
+ Arrays.Fill(_lastSkipDoc, 0);
+ Arrays.Fill(_lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list
+ for (int i = 0; i < NumberOfSkipLevels; i++)
+ {
+ _docIndex[i].CopyFrom(topDocIndex, true);
+ if (_freqOutput != null)
+ {
+ _freqIndex[i].CopyFrom(topFreqIndex, true);
+ }
+ if (_posOutput != null)
+ {
+ _posIndex[i].CopyFrom(topPosIndex, true);
+ }
+ }
+ if (_payloadOutput != null)
+ {
+ Arrays.Fill(_lastSkipPayloadPointer, _payloadOutput.FilePointer);
+ }
+ }
+
+ protected override void WriteSkipData(int level, IndexOutput skipBuffer)
+ {
+ // To efficiently store payloads in the posting lists we do not store the length of
+ // every payload. Instead we omit the length for a payload if the previous payload had
+ // the same length.
+ // However, in order to support skipping the payload length at every skip point must be known.
+ // So we use the same length encoding that we use for the posting lists for the skip data as well:
+ // Case 1: current field does not store payloads
+ // SkipDatum --> DocSkip, FreqSkip, ProxSkip
+ // DocSkip,FreqSkip,ProxSkip --> VInt
+ // DocSkip records the document number before every SkipInterval th document in TermFreqs.
+ // Document numbers are represented as differences from the previous value in the sequence.
+ // Case 2: current field stores payloads
+ // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip
+ // DocSkip,FreqSkip,ProxSkip --> VInt
+ // PayloadLength --> VInt
+ // In this case DocSkip/2 is the difference between
+ // the current and the previous value. If DocSkip
+ // is odd, then a PayloadLength encoded as VInt follows,
+ // if DocSkip is even, then it is assumed that the
+ // current payload length equals the length at the previous
+ // skip point
+
+ Debug.Assert(_indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !_curStorePayloads);
+
+ if (_curStorePayloads)
+ {
+ int delta = _curDoc - _lastSkipDoc[level];
+ if (_curPayloadLength == _lastSkipPayloadLength[level])
+ {
+ // the current payload length equals the length at the previous skip point,
+ // so we don't store the length again
+ skipBuffer.WriteVInt(delta << 1);
+ }
+ else
+ {
+ // the payload length is different from the previous one. We shift the DocSkip,
+ // set the lowest bit and store the current payload length as VInt.
+ skipBuffer.WriteVInt(delta << 1 | 1);
+ skipBuffer.WriteVInt(_curPayloadLength);
+ _lastSkipPayloadLength[level] = _curPayloadLength;
+ }
+ }
+ else
+ {
+ // current field does not store payloads
+ skipBuffer.WriteVInt(_curDoc - _lastSkipDoc[level]);
+ }
+
+ if (_indexOptions != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ _freqIndex[level].Mark();
+ _freqIndex[level].Write(skipBuffer, false);
+ }
+ _docIndex[level].Mark();
+ _docIndex[level].Write(skipBuffer, false);
+ if (_indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ _posIndex[level].Mark();
+ _posIndex[level].Write(skipBuffer, false);
+ if (_curStorePayloads)
+ {
+ skipBuffer.WriteVInt((int) (_curPayloadPointer - _lastSkipPayloadPointer[level]));
+ }
+ }
+
+ _lastSkipDoc[level] = _curDoc;
+ _lastSkipPayloadPointer[level] = _curPayloadPointer;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesReader.cs b/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesReader.cs
index 64e0b89..4ac5623 100644
--- a/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesReader.cs
+++ b/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesReader.cs
@@ -258,12 +258,10 @@ namespace Lucene.Net.Codecs.SimpleText
private Bits GetBinaryDocsWithField(FieldInfo fieldInfo)
{
var field = FIELDS[fieldInfo.Name];
- var @in = (IndexInput)DATA.Clone();
- BytesRef scratch = new BytesRef();
+ var input = (IndexInput)DATA.Clone();
+ var scratch = new BytesRef();
- DecimalFormat decoder = new DecimalFormat(field.Pattern, new DecimalFormatSymbols(Locale.ROOT));
-
- return new BitsAnonymousInnerClassHelper2(this, field, @in, scratch, decoder);
+ return new BitsAnonymousInnerClassHelper2(this, field, input, scratch);
}
private class BitsAnonymousInnerClassHelper2 : Bits
@@ -273,39 +271,32 @@ namespace Lucene.Net.Codecs.SimpleText
private readonly OneField _field;
private readonly IndexInput _input;
private readonly BytesRef _scratch;
- private readonly DecimalFormat _decoder;
public BitsAnonymousInnerClassHelper2(SimpleTextDocValuesReader outerInstance, OneField field,
- IndexInput input, BytesRef scratch, DecimalFormat decoder)
+ IndexInput input, BytesRef scratch)
{
_outerInstance = outerInstance;
_field = field;
_input = input;
_scratch = scratch;
- _decoder = decoder;
}
public bool Get(int index)
{
-
_input.Seek(_field.DataStartFilePointer + (9 + _field.Pattern.Length + _field.MaxLength + 2)*index);
SimpleTextUtil.ReadLine(_input, _scratch);
Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH));
int len;
try
{
- len =
- (int)
- _decoder.parse(new string(_scratch.bytes, _scratch.offset + SimpleTextDocValuesWriter.LENGTH.length,
- _scratch.length - LENGTH.length, StandardCharsets.UTF_8));
+ len = int.Parse(_scratch.Bytes.SubList( _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length,
+ _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString());
}
- catch (ParseException pe)
+ catch (FormatException ex)
{
- CorruptIndexException e =
- new CorruptIndexException("failed to parse int length (resource=" + _input + ")");
- e.initCause(pe);
- throw e;
+ throw new CorruptIndexException("failed to parse int value (resource=" + _input + ")", ex);
}
+
// skip past bytes
var bytes = new sbyte[len];
_input.ReadBytes(bytes, 0, len);
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesWriter.cs b/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesWriter.cs
index 81abb86..c3f8fa4 100644
--- a/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesWriter.cs
+++ b/src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesWriter.cs
@@ -83,51 +83,37 @@ namespace Lucene.Net.Codecs.SimpleText
SimpleTextUtil.Write(data, Convert.ToString(minValue), scratch);
SimpleTextUtil.WriteNewline(data);
- // build up our fixed-width "simple text packed ints"
- // format
- System.Numerics.BigInteger maxBig = System.Numerics.BigInteger.valueOf(maxValue);
- System.Numerics.BigInteger minBig = System.Numerics.BigInteger.valueOf(minValue);
- System.Numerics.BigInteger diffBig = maxBig - minBig;
- int maxBytesPerValue = diffBig.ToString().Length;
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < maxBytesPerValue; i++)
- {
- sb.Append('0');
- }
+ // build up our fixed-width "simple text packed ints" format
+ System.Numerics.BigInteger maxBig = maxValue;
+ System.Numerics.BigInteger minBig = minValue;
+ var diffBig = maxBig - minBig;
+ var maxBytesPerValue = diffBig.ToString().Length;
+ var sb = new StringBuilder();
+ for (var i = 0; i < maxBytesPerValue; i++)
+ sb.Append('0');
+
// write our pattern to the .dat
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
-//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
-//ORIGINAL LINE: final String patternString = sb.toString();
- string patternString = sb.ToString();
-
-//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
-//ORIGINAL LINE: final java.text.DecimalFormat encoder = new java.text.DecimalFormat(patternString, new java.text.DecimalFormatSymbols(java.util.Locale.ROOT));
+ var patternString = sb.ToString();
DecimalFormat encoder = new DecimalFormat(patternString, new DecimalFormatSymbols(Locale.ROOT));
int numDocsWritten = 0;
// second pass to write the values
- foreach (Number n in values)
+ foreach (var value in values)
{
- long value = n == null ? 0 : (long) n;
Debug.Assert(value >= minValue);
- Number delta = System.Numerics.BigInteger.valueOf(value) - System.Numerics.BigInteger.valueOf(minValue);
+
+ var delta = value - minValue;
string s = encoder.format(delta);
Debug.Assert(s.Length == patternString.Length);
SimpleTextUtil.Write(data, s, scratch);
SimpleTextUtil.WriteNewline(data);
- if (n == null)
- {
- SimpleTextUtil.Write(data, "F", scratch);
- }
- else
- {
- SimpleTextUtil.Write(data, "T", scratch);
- }
+ SimpleTextUtil.Write(data, n == null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
Debug.Assert(numDocsWritten <= numDocs);
@@ -141,10 +127,10 @@ namespace Lucene.Net.Codecs.SimpleText
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.BINARY);
- int maxLength = 0;
- foreach (BytesRef value in values)
+ var maxLength = 0;
+ foreach (var value in values)
{
- int length = value == null ? 0 : value.Length;
+ var length = value == null ? 0 : value.Length;
maxLength = Math.Max(maxLength, length);
}
WriteFieldEntry(field, FieldInfo.DocValuesType_e.BINARY);
@@ -179,7 +165,7 @@ namespace Lucene.Net.Codecs.SimpleText
// because it escapes:
if (value != null)
{
- data.WriteBytes(value.bytes, value.offset, value.length);
+ data.WriteBytes(value.Bytes, value.Offset, value.Length);
}
// pad to fit
@@ -188,14 +174,7 @@ namespace Lucene.Net.Codecs.SimpleText
data.WriteByte((sbyte) ' ');
}
SimpleTextUtil.WriteNewline(data);
- if (value == null)
- {
- SimpleTextUtil.Write(data, "F", scratch);
- }
- else
- {
- SimpleTextUtil.Write(data, "T", scratch);
- }
+ SimpleTextUtil.Write(data, value == null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
}
@@ -390,19 +369,18 @@ namespace Lucene.Net.Codecs.SimpleText
foreach (var n in docToOrdCount)
{
sb2.Length = 0;
- int count = (int) n;
- for (int i = 0; i < count; i++)
+ var count = (int) n;
+ for (var i = 0; i < count; i++)
{
- long ord = (long) ordStream.Next();
+ var ord = (long) ordStream.Next();
if (sb2.Length > 0)
- {
sb2.Append(",");
- }
+
sb2.Append(Convert.ToString(ord));
}
// now pad to fit: these are numbers so spaces work well. reader calls trim()
- int numPadding = maxOrdListLength - sb2.Length;
- for (int i = 0; i < numPadding; i++)
+ var numPadding = maxOrdListLength - sb2.Length;
+ for (var i = 0; i < numPadding; i++)
{
sb2.Append(' ');
}
@@ -418,7 +396,7 @@ namespace Lucene.Net.Codecs.SimpleText
try
{
Debug.Assert(_fieldsSeen.Count > 0);
- // TODO: sheisty to do this here?
+ // java : sheisty to do this here?
SimpleTextUtil.Write(data, END);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.WriteChecksum(data, scratch);
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/SimpleText/SimpleTextSegmentInfoReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/SimpleText/SimpleTextSegmentInfoReader.cs b/src/Lucene.Net.Codecs/SimpleText/SimpleTextSegmentInfoReader.cs
index e168e04..01236cd 100644
--- a/src/Lucene.Net.Codecs/SimpleText/SimpleTextSegmentInfoReader.cs
+++ b/src/Lucene.Net.Codecs/SimpleText/SimpleTextSegmentInfoReader.cs
@@ -44,7 +44,7 @@ namespace Lucene.Net.Codecs.SimpleText
public override SegmentInfo Read(Directory directory, string segmentName, IOContext context)
{
- BytesRef scratch = new BytesRef();
+ var scratch = new BytesRef();
string segFileName = IndexFileNames.SegmentFileName(segmentName, "",
SimpleTextSegmentInfoFormat.SI_EXTENSION);
ChecksumIndexInput input = directory.OpenChecksumInput(segFileName, context);
@@ -83,7 +83,7 @@ namespace Lucene.Net.Codecs.SimpleText
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_NUM_FILES));
int numFiles = Convert.ToInt32(ReadString(SimpleTextSegmentInfoWriter.SI_NUM_FILES.Length, scratch));
- HashSet<string> files = new HashSet<string>();
+ var files = new HashSet<string>();
for (int i = 0; i < numFiles; i++)
{
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/SimpleText/SimpleTextTermVectorsWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/SimpleText/SimpleTextTermVectorsWriter.cs b/src/Lucene.Net.Codecs/SimpleText/SimpleTextTermVectorsWriter.cs
index 4dd1265..b824c15 100644
--- a/src/Lucene.Net.Codecs/SimpleText/SimpleTextTermVectorsWriter.cs
+++ b/src/Lucene.Net.Codecs/SimpleText/SimpleTextTermVectorsWriter.cs
@@ -58,19 +58,19 @@ namespace Lucene.Net.Codecs.SimpleText
internal const string VECTORS_EXTENSION = "vec";
- private readonly Directory directory;
- private readonly string segment;
+ private readonly Directory _directory;
+ private readonly string _segment;
private IndexOutput _output;
- private int numDocsWritten = 0;
- private readonly BytesRef scratch = new BytesRef();
- private bool offsets;
- private bool positions;
- private bool payloads;
+ private int _numDocsWritten;
+ private readonly BytesRef _scratch = new BytesRef();
+ private bool _offsets;
+ private bool _positions;
+ private bool _payloads;
public SimpleTextTermVectorsWriter(Directory directory, string segment, IOContext context)
{
- this.directory = directory;
- this.segment = segment;
+ _directory = directory;
+ _segment = segment;
bool success = false;
try
{
@@ -89,13 +89,13 @@ namespace Lucene.Net.Codecs.SimpleText
public override void StartDocument(int numVectorFields)
{
Write(DOC);
- Write(Convert.ToString(numDocsWritten));
+ Write(Convert.ToString(_numDocsWritten));
NewLine();
Write(NUMFIELDS);
Write(Convert.ToString(numVectorFields));
NewLine();
- numDocsWritten++;
+ _numDocsWritten++;
}
public override void StartField(FieldInfo info, int numTerms, bool positions, bool offsets, bool payloads)
@@ -124,9 +124,9 @@ namespace Lucene.Net.Codecs.SimpleText
Write(Convert.ToString(numTerms));
NewLine();
- this.positions = positions;
- this.offsets = offsets;
- this.payloads = payloads;
+ _positions = positions;
+ _offsets = offsets;
+ _payloads = payloads;
}
public override void StartTerm(BytesRef term, int freq)
@@ -142,15 +142,15 @@ namespace Lucene.Net.Codecs.SimpleText
public override void AddPosition(int position, int startOffset, int endOffset, BytesRef payload)
{
- Debug.Assert(positions || offsets);
+ Debug.Assert(_positions || _offsets);
- if (positions)
+ if (_positions)
{
Write(POSITION);
Write(Convert.ToString(position));
NewLine();
- if (payloads)
+ if (_payloads)
{
Write(PAYLOAD);
if (payload != null)
@@ -162,7 +162,7 @@ namespace Lucene.Net.Codecs.SimpleText
}
}
- if (offsets)
+ if (_offsets)
{
Write(STARTOFFSET);
Write(Convert.ToString(startOffset));
@@ -174,7 +174,7 @@ namespace Lucene.Net.Codecs.SimpleText
}
}
- public override void Abort()
+ public override sealed void Abort()
{
try
{
@@ -183,22 +183,22 @@ namespace Lucene.Net.Codecs.SimpleText
finally
{
- IOUtils.DeleteFilesIgnoringExceptions(directory,
- IndexFileNames.SegmentFileName(segment, "", VECTORS_EXTENSION));
+ IOUtils.DeleteFilesIgnoringExceptions(_directory,
+ IndexFileNames.SegmentFileName(_segment, "", VECTORS_EXTENSION));
}
}
public override void Finish(FieldInfos fis, int numDocs)
{
- if (numDocsWritten != numDocs)
+ if (_numDocsWritten != numDocs)
{
throw new Exception("mergeVectors produced an invalid result: mergedDocs is " + numDocs +
- " but vec numDocs is " + numDocsWritten + " file=" + _output +
+ " but vec numDocs is " + _numDocsWritten + " file=" + _output +
"; now aborting this merge to prevent index corruption");
}
Write(END);
NewLine();
- SimpleTextUtil.WriteChecksum(_output, scratch);
+ SimpleTextUtil.WriteChecksum(_output, _scratch);
}
protected override void Dispose(bool disposing)
@@ -222,7 +222,7 @@ namespace Lucene.Net.Codecs.SimpleText
private void Write(string s)
{
- SimpleTextUtil.Write(_output, s, scratch);
+ SimpleTextUtil.Write(_output, s, _scratch);
}
private void Write(BytesRef bytes)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/StringHelperClass.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/StringHelperClass.cs b/src/Lucene.Net.Codecs/StringHelperClass.cs
index a9ba97a..0e333f3 100644
--- a/src/Lucene.Net.Codecs/StringHelperClass.cs
+++ b/src/Lucene.Net.Codecs/StringHelperClass.cs
@@ -22,7 +22,9 @@
// This class is used to convert some aspects of the Java String class.
//-------------------------------------------------------------------------------------------
-namespace Lucene.Net.Codes
+using System.Runtime.InteropServices;
+
+namespace Lucene.Net.Codecs
{
internal static class StringHelperClass
@@ -109,7 +111,7 @@ namespace Lucene.Net.Codes
private static sbyte[] GetSBytesForEncoding(System.Text.Encoding encoding, string s)
{
- sbyte[] sbytes = new sbyte[encoding.GetByteCount(s)];
+ var sbytes = new sbyte[encoding.GetByteCount(s)];
encoding.GetBytes(s, 0, s.Length, (byte[]) (object) sbytes, 0);
return sbytes;
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Core/Codecs/StoredFieldsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/StoredFieldsReader.cs b/src/Lucene.Net.Core/Codecs/StoredFieldsReader.cs
index caa8403..997e530 100644
--- a/src/Lucene.Net.Core/Codecs/StoredFieldsReader.cs
+++ b/src/Lucene.Net.Core/Codecs/StoredFieldsReader.cs
@@ -1,31 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
using System;
namespace Lucene.Net.Codecs
{
- /// <summary>
- /// Copyright 2004 The Apache Software Foundation
- ///
- /// Licensed under the Apache License, Version 2.0 (the "License"); you may not
- /// use this file except in compliance with the License. You may obtain a copy of
- /// the License at
- ///
- /// http://www.apache.org/licenses/LICENSE-2.0
- ///
- /// Unless required by applicable law or agreed to in writing, software
- /// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- /// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- /// License for the specific language governing permissions and limitations under
- /// the License.
- /// </summary>
-
- using StoredFieldVisitor = Lucene.Net.Index.StoredFieldVisitor;
+ using StoredFieldVisitor = Index.StoredFieldVisitor;
/// <summary>
/// Codec API for reading stored fields.
- /// <p>
- /// You need to implement <seealso cref="#visitDocument(int, StoredFieldVisitor)"/> to
- /// read the stored fields for a document, implement <seealso cref="#clone()"/> (creating
- /// clones of any IndexInputs used, etc), and <seealso cref="#close()"/>
+ ///
+ /// You need to implement visitDocument(int, StoredFieldVisitor) to
+ /// read the stored fields for a document, implement clone( (creating
+ /// clones of any IndexInputs used, etc), and close()
+ ///
/// @lucene.experimental
/// </summary>
public abstract class StoredFieldsReader : ICloneable, IDisposable
@@ -58,7 +60,7 @@ namespace Lucene.Net.Codecs
/// <summary>
/// Checks consistency of this reader.
- /// <p>
+ ///
/// Note that this may be costly in terms of I/O, e.g.
/// may involve computing a checksum value against large data files.
/// @lucene.internal
[05/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
index 285c243..26ee255 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
@@ -1,408 +1,548 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import codecs.CodecUtil;
-import codecs.DocValuesConsumer;
-import index.FieldInfo;
-import index.IndexFileNames;
-import index.SegmentWriteState;
-import store.ByteArrayDataOutput;
-import store.IndexOutput;
-import util.ArrayUtil;
-import util.BytesRef;
-import util.IOUtils;
-import util.IntsRef;
-import util.MathUtil;
-import util.fst.Builder;
-import util.fst.FST.INPUT_TYPE;
-import util.fst.FST;
-import util.fst.PositiveIntOutputs;
-import util.fst.Util;
-import util.packed.BlockPackedWriter;
-import util.packed.MonotonicBlockPackedWriter;
-import util.packed.PackedInts.FormatAndBits;
-import util.packed.PackedInts;
-
-import static codecs.memory.MemoryDocValuesProducer.VERSION_CURRENT;
-import static codecs.memory.MemoryDocValuesProducer.BLOCK_SIZE;
-import static codecs.memory.MemoryDocValuesProducer.BYTES;
-import static codecs.memory.MemoryDocValuesProducer.NUMBER;
-import static codecs.memory.MemoryDocValuesProducer.FST;
-import static codecs.memory.MemoryDocValuesProducer.DELTA_COMPRESSED;
-import static codecs.memory.MemoryDocValuesProducer.GCD_COMPRESSED;
-import static codecs.memory.MemoryDocValuesProducer.TABLE_COMPRESSED;
-import static codecs.memory.MemoryDocValuesProducer.UNCOMPRESSED;
-
-/**
- * Writer for {@link MemoryDocValuesFormat}
- */
-class MemoryDocValuesConsumer extends DocValuesConsumer {
- IndexOutput data, meta;
- final int maxDoc;
- final float acceptableOverheadRatio;
-
- MemoryDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension, float acceptableOverheadRatio) {
- this.acceptableOverheadRatio = acceptableOverheadRatio;
- maxDoc = state.segmentInfo.getDocCount();
- bool success = false;
- try {
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- @Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) {
- addNumericField(field, values, true);
- }
-
- void addNumericField(FieldInfo field, Iterable<Number> values, bool optimizeStorage) {
- meta.writeVInt(field.number);
- meta.writeByte(NUMBER);
- meta.writeLong(data.getFilePointer());
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
- long gcd = 0;
- bool missing = false;
- // TODO: more efficient?
- HashSet<Long> uniqueValues = null;
- if (optimizeStorage) {
- uniqueValues = new HashSet<>();
-
- long count = 0;
- for (Number nv : values) {
- final long v;
- if (nv == null) {
- v = 0;
- missing = true;
- } else {
- v = nv.longValue();
- }
-
- if (gcd != 1) {
- if (v < Long.MIN_VALUE / 2 || v > Long.MAX_VALUE / 2) {
- // in that case v - minValue might overflow and make the GCD computation return
- // wrong results. Since these extreme values are unlikely, we just discard
- // GCD computation for them
- gcd = 1;
- } else if (count != 0) { // minValue needs to be set first
- gcd = MathUtil.gcd(gcd, v - minValue);
- }
- }
-
- minValue = Math.min(minValue, v);
- maxValue = Math.max(maxValue, v);
-
- if (uniqueValues != null) {
- if (uniqueValues.add(v)) {
- if (uniqueValues.size() > 256) {
- uniqueValues = null;
- }
- }
- }
-
- ++count;
- }
- Debug.Assert( count == maxDoc;
- }
-
- if (missing) {
- long start = data.getFilePointer();
- writeMissingBitset(values);
- meta.writeLong(start);
- meta.writeLong(data.getFilePointer() - start);
- } else {
- meta.writeLong(-1L);
- }
-
- if (uniqueValues != null) {
- // small number of unique values
- final int bitsPerValue = PackedInts.bitsRequired(uniqueValues.size()-1);
- FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits(maxDoc, bitsPerValue, acceptableOverheadRatio);
- if (formatAndBits.bitsPerValue == 8 && minValue >= Byte.MIN_VALUE && maxValue <= Byte.MAX_VALUE) {
- meta.writeByte(UNCOMPRESSED); // uncompressed
- for (Number nv : values) {
- data.writeByte(nv == null ? 0 : (byte) nv.longValue());
- }
- } else {
- meta.writeByte(TABLE_COMPRESSED); // table-compressed
- Long[] decode = uniqueValues.toArray(new Long[uniqueValues.size()]);
- final HashMap<Long,Integer> encode = new HashMap<>();
- data.writeVInt(decode.length);
- for (int i = 0; i < decode.length; i++) {
- data.writeLong(decode[i]);
- encode.put(decode[i], i);
- }
-
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- data.writeVInt(formatAndBits.format.getId());
- data.writeVInt(formatAndBits.bitsPerValue);
-
- final PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, formatAndBits.format, maxDoc, formatAndBits.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE);
- for(Number nv : values) {
- writer.add(encode.get(nv == null ? 0 : nv.longValue()));
- }
- writer.finish();
- }
- } else if (gcd != 0 && gcd != 1) {
- meta.writeByte(GCD_COMPRESSED);
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- data.writeLong(minValue);
- data.writeLong(gcd);
- data.writeVInt(BLOCK_SIZE);
-
- final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE);
- for (Number nv : values) {
- long value = nv == null ? 0 : nv.longValue();
- writer.add((value - minValue) / gcd);
- }
- writer.finish();
- } else {
- meta.writeByte(DELTA_COMPRESSED); // delta-compressed
-
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- data.writeVInt(BLOCK_SIZE);
-
- final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE);
- for (Number nv : values) {
- writer.add(nv == null ? 0 : nv.longValue());
- }
- writer.finish();
- }
- }
-
- @Override
- public void close() {
- bool success = false;
- try {
- if (meta != null) {
- meta.writeVInt(-1); // write EOF marker
- CodecUtil.writeFooter(meta); // write checksum
- }
- if (data != null) {
- CodecUtil.writeFooter(data);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(data, meta);
- } else {
- IOUtils.closeWhileHandlingException(data, meta);
- }
- data = meta = null;
- }
- }
-
- @Override
- public void addBinaryField(FieldInfo field, final Iterable<BytesRef> values) {
- // write the byte[] data
- meta.writeVInt(field.number);
- meta.writeByte(BYTES);
- int minLength = Integer.MAX_VALUE;
- int maxLength = Integer.MIN_VALUE;
- final long startFP = data.getFilePointer();
- bool missing = false;
- for(BytesRef v : values) {
- final int length;
- if (v == null) {
- length = 0;
- missing = true;
- } else {
- length = v.length;
- }
- if (length > MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH) {
- throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH);
- }
- minLength = Math.min(minLength, length);
- maxLength = Math.max(maxLength, length);
- if (v != null) {
- data.writeBytes(v.bytes, v.offset, v.length);
- }
- }
- meta.writeLong(startFP);
- meta.writeLong(data.getFilePointer() - startFP);
- if (missing) {
- long start = data.getFilePointer();
- writeMissingBitset(values);
- meta.writeLong(start);
- meta.writeLong(data.getFilePointer() - start);
- } else {
- meta.writeLong(-1L);
- }
- meta.writeVInt(minLength);
- meta.writeVInt(maxLength);
-
- // if minLength == maxLength, its a fixed-length byte[], we are done (the addresses are implicit)
- // otherwise, we need to record the length fields...
- if (minLength != maxLength) {
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeVInt(BLOCK_SIZE);
-
- final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
- long addr = 0;
- for (BytesRef v : values) {
- if (v != null) {
- addr += v.length;
- }
- writer.add(addr);
- }
- writer.finish();
- }
- }
-
- private void writeFST(FieldInfo field, Iterable<BytesRef> values) {
- meta.writeVInt(field.number);
- meta.writeByte(FST);
- meta.writeLong(data.getFilePointer());
- PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
- Builder<Long> builder = new Builder<>(INPUT_TYPE.BYTE1, outputs);
- IntsRef scratch = new IntsRef();
- long ord = 0;
- for (BytesRef v : values) {
- builder.add(Util.toIntsRef(v, scratch), ord);
- ord++;
- }
- FST<Long> fst = builder.finish();
- if (fst != null) {
- fst.save(data);
- }
- meta.writeVLong(ord);
- }
-
- // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on,
- // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode)
- void writeMissingBitset(Iterable<?> values) {
- long bits = 0;
- int count = 0;
- for (Object v : values) {
- if (count == 64) {
- data.writeLong(bits);
- count = 0;
- bits = 0;
- }
- if (v != null) {
- bits |= 1L << (count & 0x3f);
- }
- count++;
- }
- if (count > 0) {
- data.writeLong(bits);
- }
- }
-
- @Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) {
- // write the ordinals as numerics
- addNumericField(field, docToOrd, false);
-
- // write the values as FST
- writeFST(field, values);
- }
-
- // note: this might not be the most efficient... but its fairly simple
- @Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) {
- // write the ordinals as a binary field
- addBinaryField(field, new Iterable<BytesRef>() {
- @Override
- public Iterator<BytesRef> iterator() {
- return new SortedSetIterator(docToOrdCount.iterator(), ords.iterator());
- }
- });
-
- // write the values as FST
- writeFST(field, values);
- }
-
- // per-document vint-encoded byte[]
- static class SortedSetIterator implements Iterator<BytesRef> {
- byte[] buffer = new byte[10];
- ByteArrayDataOutput out = new ByteArrayDataOutput();
- BytesRef ref = new BytesRef();
-
- final Iterator<Number> counts;
- final Iterator<Number> ords;
-
- SortedSetIterator(Iterator<Number> counts, Iterator<Number> ords) {
- this.counts = counts;
- this.ords = ords;
- }
-
- @Override
- public bool hasNext() {
- return counts.hasNext();
- }
-
- @Override
- public BytesRef next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- int count = counts.next().intValue();
- int maxSize = count*9; // worst case
- if (maxSize > buffer.length) {
- buffer = ArrayUtil.grow(buffer, maxSize);
- }
-
- try {
- encodeValues(count);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
-
- ref.bytes = buffer;
- ref.offset = 0;
- ref.length = out.getPosition();
-
- return ref;
- }
-
- // encodes count values to buffer
- private void encodeValues(int count) {
- out.reset(buffer);
- long lastOrd = 0;
- for (int i = 0; i < count; i++) {
- long ord = ords.next().longValue();
- out.writeVLong(ord - lastOrd);
- lastOrd = ord;
- }
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-}
+using System;
+using System.Diagnostics;
+using System.Collections.Generic;
+using Lucene.Net.Codecs.Memory;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using ByteArrayDataOutput = org.apache.lucene.store.ByteArrayDataOutput;
+ using IndexOutput = org.apache.lucene.store.IndexOutput;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using IntsRef = org.apache.lucene.util.IntsRef;
+ using MathUtil = org.apache.lucene.util.MathUtil;
+ using Builder = org.apache.lucene.util.fst.Builder;
+ using INPUT_TYPE = org.apache.lucene.util.fst.FST.INPUT_TYPE;
+ using FST = org.apache.lucene.util.fst.FST;
+ using PositiveIntOutputs = org.apache.lucene.util.fst.PositiveIntOutputs;
+ using Util = org.apache.lucene.util.fst.Util;
+ using BlockPackedWriter = org.apache.lucene.util.packed.BlockPackedWriter;
+ using MonotonicBlockPackedWriter = org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
+ using FormatAndBits = org.apache.lucene.util.packed.PackedInts.FormatAndBits;
+ using PackedInts = org.apache.lucene.util.packed.PackedInts;
+
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.VERSION_CURRENT;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BLOCK_SIZE;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BYTES;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.NUMBER;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.FST;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.DELTA_COMPRESSED;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.GCD_COMPRESSED;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.TABLE_COMPRESSED;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.UNCOMPRESSED;
+
+ /// <summary>
+ /// Writer for <seealso cref="MemoryDocValuesFormat"/>
+ /// </summary>
+ internal class MemoryDocValuesConsumer : DocValuesConsumer
+ {
+ internal IndexOutput data, meta;
+ internal readonly int maxDoc;
+ internal readonly float acceptableOverheadRatio;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: MemoryDocValuesConsumer(org.apache.lucene.index.SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension, float acceptableOverheadRatio) throws java.io.IOException
+ internal MemoryDocValuesConsumer(SegmentWriteState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension, float acceptableOverheadRatio)
+ {
+ this.acceptableOverheadRatio = acceptableOverheadRatio;
+ maxDoc = state.segmentInfo.DocCount;
+ bool success = false;
+ try
+ {
+ string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+ data = state.directory.createOutput(dataName, state.context);
+ CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+ meta = state.directory.createOutput(metaName, state.context);
+ CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(this);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addNumericField(org.apache.lucene.index.FieldInfo field, Iterable<Number> values) throws java.io.IOException
+ public override void addNumericField(FieldInfo field, IEnumerable<Number> values)
+ {
+ addNumericField(field, values, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void addNumericField(org.apache.lucene.index.FieldInfo field, Iterable<Number> values, boolean optimizeStorage) throws java.io.IOException
+ internal virtual void addNumericField(FieldInfo field, IEnumerable<Number> values, bool optimizeStorage)
+ {
+ meta.writeVInt(field.number);
+ meta.writeByte(NUMBER);
+ meta.writeLong(data.FilePointer);
+ long minValue = long.MaxValue;
+ long maxValue = long.MinValue;
+ long gcd = 0;
+ bool missing = false;
+ // TODO: more efficient?
+ HashSet<long?> uniqueValues = null;
+ if (optimizeStorage)
+ {
+ uniqueValues = new HashSet<>();
+
+ long count = 0;
+ foreach (Number nv in values)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long v;
+ long v;
+ if (nv == null)
+ {
+ v = 0;
+ missing = true;
+ }
+ else
+ {
+ v = (long)nv;
+ }
+
+ if (gcd != 1)
+ {
+ if (v < long.MinValue / 2 || v > long.MaxValue / 2)
+ {
+ // in that case v - minValue might overflow and make the GCD computation return
+ // wrong results. Since these extreme values are unlikely, we just discard
+ // GCD computation for them
+ gcd = 1;
+ } // minValue needs to be set first
+ else if (count != 0)
+ {
+ gcd = MathUtil.gcd(gcd, v - minValue);
+ }
+ }
+
+ minValue = Math.Min(minValue, v);
+ maxValue = Math.Max(maxValue, v);
+
+ if (uniqueValues != null)
+ {
+ if (uniqueValues.Add(v))
+ {
+ if (uniqueValues.Count > 256)
+ {
+ uniqueValues = null;
+ }
+ }
+ }
+
+ ++count;
+ }
+ Debug.Assert(count == maxDoc);
+ }
+
+ if (missing)
+ {
+ long start = data.FilePointer;
+ writeMissingBitset(values);
+ meta.writeLong(start);
+ meta.writeLong(data.FilePointer - start);
+ }
+ else
+ {
+ meta.writeLong(-1L);
+ }
+
+ if (uniqueValues != null)
+ {
+ // small number of unique values
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int bitsPerValue = org.apache.lucene.util.packed.PackedInts.bitsRequired(uniqueValues.size()-1);
+ int bitsPerValue = PackedInts.bitsRequired(uniqueValues.Count - 1);
+ FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits(maxDoc, bitsPerValue, acceptableOverheadRatio);
+ if (formatAndBits.bitsPerValue == 8 && minValue >= sbyte.MinValue && maxValue <= sbyte.MaxValue)
+ {
+ meta.writeByte(UNCOMPRESSED); // uncompressed
+ foreach (Number nv in values)
+ {
+ data.writeByte(nv == null ? 0 : (long)(sbyte) nv);
+ }
+ }
+ else
+ {
+ meta.writeByte(TABLE_COMPRESSED); // table-compressed
+ long?[] decode = uniqueValues.toArray(new long?[uniqueValues.Count]);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.HashMap<Long,Integer> encode = new java.util.HashMap<>();
+ Dictionary<long?, int?> encode = new Dictionary<long?, int?>();
+ data.writeVInt(decode.Length);
+ for (int i = 0; i < decode.Length; i++)
+ {
+ data.writeLong(decode[i]);
+ encode[decode[i]] = i;
+ }
+
+ meta.writeVInt(PackedInts.VERSION_CURRENT);
+ data.writeVInt(formatAndBits.format.Id);
+ data.writeVInt(formatAndBits.bitsPerValue);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.PackedInts.Writer writer = org.apache.lucene.util.packed.PackedInts.getWriterNoHeader(data, formatAndBits.format, maxDoc, formatAndBits.bitsPerValue, org.apache.lucene.util.packed.PackedInts.DEFAULT_BUFFER_SIZE);
+ PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, formatAndBits.format, maxDoc, formatAndBits.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE);
+ foreach (Number nv in values)
+ {
+ writer.add(encode[nv == null ? 0 : (long)nv]);
+ }
+ writer.finish();
+ }
+ }
+ else if (gcd != 0 && gcd != 1)
+ {
+ meta.writeByte(GCD_COMPRESSED);
+ meta.writeVInt(PackedInts.VERSION_CURRENT);
+ data.writeLong(minValue);
+ data.writeLong(gcd);
+ data.writeVInt(BLOCK_SIZE);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.BlockPackedWriter writer = new org.apache.lucene.util.packed.BlockPackedWriter(data, BLOCK_SIZE);
+ BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE);
+ foreach (Number nv in values)
+ {
+ long value = nv == null ? 0 : (long)nv;
+ writer.add((value - minValue) / gcd);
+ }
+ writer.finish();
+ }
+ else
+ {
+ meta.writeByte(DELTA_COMPRESSED); // delta-compressed
+
+ meta.writeVInt(PackedInts.VERSION_CURRENT);
+ data.writeVInt(BLOCK_SIZE);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.BlockPackedWriter writer = new org.apache.lucene.util.packed.BlockPackedWriter(data, BLOCK_SIZE);
+ BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE);
+ foreach (Number nv in values)
+ {
+ writer.add(nv == null ? 0 : (long)nv);
+ }
+ writer.finish();
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ bool success = false;
+ try
+ {
+ if (meta != null)
+ {
+ meta.writeVInt(-1); // write EOF marker
+ CodecUtil.writeFooter(meta); // write checksum
+ }
+ if (data != null)
+ {
+ CodecUtil.writeFooter(data);
+ }
+ success = true;
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.close(data, meta);
+ }
+ else
+ {
+ IOUtils.closeWhileHandlingException(data, meta);
+ }
+ data = meta = null;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addBinaryField(org.apache.lucene.index.FieldInfo field, final Iterable<org.apache.lucene.util.BytesRef> values) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ public override void addBinaryField(FieldInfo field, IEnumerable<BytesRef> values)
+ {
+ // write the byte[] data
+ meta.writeVInt(field.number);
+ meta.writeByte(BYTES);
+ int minLength = int.MaxValue;
+ int maxLength = int.MinValue;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long startFP = data.getFilePointer();
+ long startFP = data.FilePointer;
+ bool missing = false;
+ foreach (BytesRef v in values)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int length;
+ int length;
+ if (v == null)
+ {
+ length = 0;
+ missing = true;
+ }
+ else
+ {
+ length = v.length;
+ }
+ if (length > MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH)
+ {
+ throw new System.ArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH);
+ }
+ minLength = Math.Min(minLength, length);
+ maxLength = Math.Max(maxLength, length);
+ if (v != null)
+ {
+ data.writeBytes(v.bytes, v.offset, v.length);
+ }
+ }
+ meta.writeLong(startFP);
+ meta.writeLong(data.FilePointer - startFP);
+ if (missing)
+ {
+ long start = data.FilePointer;
+ writeMissingBitset(values);
+ meta.writeLong(start);
+ meta.writeLong(data.FilePointer - start);
+ }
+ else
+ {
+ meta.writeLong(-1L);
+ }
+ meta.writeVInt(minLength);
+ meta.writeVInt(maxLength);
+
+ // if minLength == maxLength, its a fixed-length byte[], we are done (the addresses are implicit)
+ // otherwise, we need to record the length fields...
+ if (minLength != maxLength)
+ {
+ meta.writeVInt(PackedInts.VERSION_CURRENT);
+ meta.writeVInt(BLOCK_SIZE);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.MonotonicBlockPackedWriter writer = new org.apache.lucene.util.packed.MonotonicBlockPackedWriter(data, BLOCK_SIZE);
+ MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
+ long addr = 0;
+ foreach (BytesRef v in values)
+ {
+ if (v != null)
+ {
+ addr += v.length;
+ }
+ writer.add(addr);
+ }
+ writer.finish();
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void writeFST(org.apache.lucene.index.FieldInfo field, Iterable<org.apache.lucene.util.BytesRef> values) throws java.io.IOException
+ private void writeFST(FieldInfo field, IEnumerable<BytesRef> values)
+ {
+ meta.writeVInt(field.number);
+ meta.writeByte(FST);
+ meta.writeLong(data.FilePointer);
+ PositiveIntOutputs outputs = PositiveIntOutputs.Singleton;
+ Builder<long?> builder = new Builder<long?>(INPUT_TYPE.BYTE1, outputs);
+ IntsRef scratch = new IntsRef();
+ long ord = 0;
+ foreach (BytesRef v in values)
+ {
+ builder.add(Util.toIntsRef(v, scratch), ord);
+ ord++;
+ }
+ FST<long?> fst = builder.finish();
+ if (fst != null)
+ {
+ fst.save(data);
+ }
+ meta.writeVLong(ord);
+ }
+
+ // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on,
+ // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode)
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void writeMissingBitset(Iterable<?> values) throws java.io.IOException
+ internal virtual void writeMissingBitset<T1>(IEnumerable<T1> values)
+ {
+ long bits = 0;
+ int count = 0;
+ foreach (object v in values)
+ {
+ if (count == 64)
+ {
+ data.writeLong(bits);
+ count = 0;
+ bits = 0;
+ }
+ if (v != null)
+ {
+ bits |= 1L << (count & 0x3f);
+ }
+ count++;
+ }
+ if (count > 0)
+ {
+ data.writeLong(bits);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addSortedField(org.apache.lucene.index.FieldInfo field, Iterable<org.apache.lucene.util.BytesRef> values, Iterable<Number> docToOrd) throws java.io.IOException
+ public override void addSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<Number> docToOrd)
+ {
+ // write the ordinals as numerics
+ addNumericField(field, docToOrd, false);
+
+ // write the values as FST
+ writeFST(field, values);
+ }
+
+ // note: this might not be the most efficient... but its fairly simple
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addSortedSetField(org.apache.lucene.index.FieldInfo field, Iterable<org.apache.lucene.util.BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ public override void addSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<Number> docToOrdCount, IEnumerable<Number> ords)
+ {
+ // write the ordinals as a binary field
+ addBinaryField(field, new IterableAnonymousInnerClassHelper(this, docToOrdCount, ords));
+
+ // write the values as FST
+ writeFST(field, values);
+ }
+
+ private class IterableAnonymousInnerClassHelper : IEnumerable<BytesRef>
+ {
+ private readonly MemoryDocValuesConsumer outerInstance;
+
+ private IEnumerable<Number> docToOrdCount;
+ private IEnumerable<Number> ords;
+
+ public IterableAnonymousInnerClassHelper(MemoryDocValuesConsumer outerInstance, IEnumerable<Number> docToOrdCount, IEnumerable<Number> ords)
+ {
+ this.outerInstance = outerInstance;
+ this.docToOrdCount = docToOrdCount;
+ this.ords = ords;
+ }
+
+ public virtual IEnumerator<BytesRef> GetEnumerator()
+ {
+ return new SortedSetIterator(docToOrdCount.GetEnumerator(), ords.GetEnumerator());
+ }
+ }
+
+ // per-document vint-encoded byte[]
+ internal class SortedSetIterator : IEnumerator<BytesRef>
+ {
+ internal sbyte[] buffer = new sbyte[10];
+ internal ByteArrayDataOutput @out = new ByteArrayDataOutput();
+ internal BytesRef @ref = new BytesRef();
+
+ internal readonly IEnumerator<Number> counts;
+ internal readonly IEnumerator<Number> ords;
+
+ internal SortedSetIterator(IEnumerator<Number> counts, IEnumerator<Number> ords)
+ {
+ this.counts = counts;
+ this.ords = ords;
+ }
+
+ public override bool hasNext()
+ {
+//JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops:
+ return counts.hasNext();
+ }
+
+ public override BytesRef next()
+ {
+ if (!hasNext())
+ {
+ throw new NoSuchElementException();
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops:
+ int count = (int)counts.next();
+ int maxSize = count * 9; // worst case
+ if (maxSize > buffer.Length)
+ {
+ buffer = ArrayUtil.grow(buffer, maxSize);
+ }
+
+ try
+ {
+ encodeValues(count);
+ }
+ catch (IOException bogus)
+ {
+ throw new Exception(bogus);
+ }
+
+ @ref.bytes = buffer;
+ @ref.offset = 0;
+ @ref.length = @out.Position;
+
+ return @ref;
+ }
+
+ // encodes count values to buffer
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void encodeValues(int count) throws java.io.IOException
+ internal virtual void encodeValues(int count)
+ {
+ @out.reset(buffer);
+ long lastOrd = 0;
+ for (int i = 0; i < count; i++)
+ {
+//JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops:
+ long ord = (long)ords.next();
+ @out.writeVLong(ord - lastOrd);
+ lastOrd = ord;
+ }
+ }
+
+ public override void remove()
+ {
+ throw new System.NotSupportedException();
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/MemoryDocValuesFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesFormat.cs b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesFormat.cs
index 34b360c..8b36c0f 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesFormat.cs
@@ -1,72 +1,73 @@
-package codecs.memory;
+using org.apache.lucene.codecs.memory;
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+namespace Lucene.Net.Codecs.Memory
+{
-import java.io.IOException;
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ /// <summary>
+ /// In-memory docvalues format </summary>
+ public class MemoryDocValuesFormat : DocValuesFormat
+ {
-import codecs.DocValuesConsumer;
-import codecs.DocValuesProducer;
-import codecs.DocValuesFormat;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import util.packed.PackedInts;
+ /// <summary>
+ /// Maximum length for each binary doc values field. </summary>
+ public static readonly int MAX_BINARY_FIELD_LENGTH = (1 << 15) - 2;
-/** In-memory docvalues format */
-public class MemoryDocValuesFormat extends DocValuesFormat {
+ internal readonly float acceptableOverheadRatio;
- /** Maximum length for each binary doc values field. */
- public static final int MAX_BINARY_FIELD_LENGTH = (1 << 15) - 2;
-
- final float acceptableOverheadRatio;
-
- /**
- * Calls {@link #MemoryDocValuesFormat(float)
- * MemoryDocValuesFormat(PackedInts.DEFAULT)}
- */
- public MemoryDocValuesFormat() {
- this(PackedInts.DEFAULT);
- }
-
- /**
- * Creates a new MemoryDocValuesFormat with the specified
- * <code>acceptableOverheadRatio</code> for NumericDocValues.
- * @param acceptableOverheadRatio compression parameter for numerics.
- * Currently this is only used when the number of unique values is small.
- *
- * @lucene.experimental
- */
- public MemoryDocValuesFormat(float acceptableOverheadRatio) {
- super("Memory");
- this.acceptableOverheadRatio = acceptableOverheadRatio;
- }
+ /// <summary>
+ /// Calls {@link #MemoryDocValuesFormat(float)
+ /// MemoryDocValuesFormat(PackedInts.DEFAULT)}
+ /// </summary>
+ public MemoryDocValuesFormat() : this(PackedInts.DEFAULT)
+ {
+ }
- @Override
- public DocValuesConsumer fieldsConsumer(SegmentWriteState state) {
- return new MemoryDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION, acceptableOverheadRatio);
- }
-
- @Override
- public DocValuesProducer fieldsProducer(SegmentReadState state) {
- return new MemoryDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
- }
-
- static final String DATA_CODEC = "MemoryDocValuesData";
- static final String DATA_EXTENSION = "mdvd";
- static final String METADATA_CODEC = "MemoryDocValuesMetadata";
- static final String METADATA_EXTENSION = "mdvm";
-}
+ /// <summary>
+ /// Creates a new MemoryDocValuesFormat with the specified
+ /// <code>acceptableOverheadRatio</code> for NumericDocValues. </summary>
+ /// <param name="acceptableOverheadRatio"> compression parameter for numerics.
+ /// Currently this is only used when the number of unique values is small.
+ ///
+ /// @lucene.experimental </param>
+ public MemoryDocValuesFormat(float acceptableOverheadRatio) : base("Memory")
+ {
+ this.acceptableOverheadRatio = acceptableOverheadRatio;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.DocValuesConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override DocValuesConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ return new MemoryDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION, acceptableOverheadRatio);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.DocValuesProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override DocValuesProducer fieldsProducer(SegmentReadState state)
+ {
+ return new MemoryDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
+
+ internal const string DATA_CODEC = "MemoryDocValuesData";
+ internal const string DATA_EXTENSION = "mdvd";
+ internal const string METADATA_CODEC = "MemoryDocValuesMetadata";
+ internal const string METADATA_EXTENSION = "mdvm";
+ }
+
+}
\ No newline at end of file
[07/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTOrdTermsWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsWriter.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsWriter.cs
index 76d0092..51c7278 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsWriter.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsWriter.cs
@@ -1,374 +1,435 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Comparator;
-
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.SegmentWriteState;
-import store.DataOutput;
-import store.IndexOutput;
-import store.RAMOutputStream;
-import util.ArrayUtil;
-import util.BytesRef;
-import util.IOUtils;
-import util.IntsRef;
-import util.fst.Builder;
-import util.fst.FST;
-import util.fst.PositiveIntOutputs;
-import util.fst.Util;
-import codecs.BlockTermState;
-import codecs.PostingsWriterBase;
-import codecs.PostingsConsumer;
-import codecs.FieldsConsumer;
-import codecs.TermsConsumer;
-import codecs.TermStats;
-import codecs.CodecUtil;
-
-/**
- * FST-based term dict, using ord as FST output.
- *
- * The FST holds the mapping between <term, ord>, and
- * term's metadata is delta encoded into a single byte block.
- *
- * Typically the byte block consists of four parts:
- * 1. term statistics: docFreq, totalTermFreq;
- * 2. monotonic long[], e.g. the pointer to the postings list for that term;
- * 3. generic byte[], e.g. other information customized by postings base.
- * 4. single-level skip list to speed up metadata decoding by ord.
- *
- * <p>
- * Files:
- * <ul>
- * <li><tt>.tix</tt>: <a href="#Termindex">Term Index</a></li>
- * <li><tt>.tbk</tt>: <a href="#Termblock">Term Block</a></li>
- * </ul>
- * </p>
- *
- * <a name="Termindex" id="Termindex"></a>
- * <h3>Term Index</h3>
- * <p>
- * The .tix contains a list of FSTs, one for each field.
- * The FST maps a term to its corresponding order in current field.
- * </p>
- *
- * <ul>
- * <li>TermIndex(.tix) --> Header, TermFST<sup>NumFields</sup>, Footer</li>
- * <li>TermFST --> {@link FST FST<long>}</li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
- * </ul>
- *
- * <p>Notes:</p>
- * <ul>
- * <li>
- * Since terms are already sorted before writing to <a href="#Termblock">Term Block</a>,
- * their ords can directly used to seek term metadata from term block.
- * </li>
- * </ul>
- *
- * <a name="Termblock" id="Termblock"></a>
- * <h3>Term Block</h3>
- * <p>
- * The .tbk contains all the statistics and metadata for terms, along with field summary (e.g.
- * per-field data like number of documents in current field). For each field, there are four blocks:
- * <ul>
- * <li>statistics bytes block: contains term statistics; </li>
- * <li>metadata longs block: delta-encodes monotonic part of metadata; </li>
- * <li>metadata bytes block: encodes other parts of metadata; </li>
- * <li>skip block: contains skip data, to speed up metadata seeking and decoding</li>
- * </ul>
- * </p>
- *
- * <p>File Format:</p>
- * <ul>
- * <li>TermBlock(.tbk) --> Header, <i>PostingsHeader</i>, FieldSummary, DirOffset</li>
- * <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, SumTotalTermFreq?, SumDocFreq,
- * DocCount, LongsSize, DataBlock > <sup>NumFields</sup>, Footer</li>
- *
- * <li>DataBlock --> StatsBlockLength, MetaLongsBlockLength, MetaBytesBlockLength,
- * SkipBlock, StatsBlock, MetaLongsBlock, MetaBytesBlock </li>
- * <li>SkipBlock --> < StatsFPDelta, MetaLongsSkipFPDelta, MetaBytesSkipFPDelta,
- * MetaLongsSkipDelta<sup>LongsSize</sup> ><sup>NumTerms</sup>
- * <li>StatsBlock --> < DocFreq[Same?], (TotalTermFreq-DocFreq) ? > <sup>NumTerms</sup>
- * <li>MetaLongsBlock --> < LongDelta<sup>LongsSize</sup>, BytesSize > <sup>NumTerms</sup>
- * <li>MetaBytesBlock --> Byte <sup>MetaBytesBlockLength</sup>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
- * <li>NumFields, FieldNumber, DocCount, DocFreq, LongsSize,
- * FieldNumber, DocCount --> {@link DataOutput#writeVInt VInt}</li>
- * <li>NumTerms, SumTotalTermFreq, SumDocFreq, StatsBlockLength, MetaLongsBlockLength, MetaBytesBlockLength,
- * StatsFPDelta, MetaLongsSkipFPDelta, MetaBytesSkipFPDelta, MetaLongsSkipStart, TotalTermFreq,
- * LongDelta,--> {@link DataOutput#writeVLong VLong}</li>
- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
- * </ul>
- * <p>Notes: </p>
- * <ul>
- * <li>
- * The format of PostingsHeader and MetaBytes are customized by the specific postings implementation:
- * they contain arbitrary per-file data (such as parameters or versioning information), and per-term data
- * (non-monotonic ones like pulsed postings data).
- * </li>
- * <li>
- * During initialization the reader will load all the blocks into memory. SkipBlock will be decoded, so that during seek
- * term dict can lookup file pointers directly. StatsFPDelta, MetaLongsSkipFPDelta, etc. are file offset
- * for every SkipInterval's term. MetaLongsSkipDelta is the difference from previous one, which indicates
- * the value of preceding metadata longs for every SkipInterval's term.
- * </li>
- * <li>
- * DocFreq is the count of documents which contain the term. TotalTermFreq is the total number of occurrences of the term.
- * Usually these two values are the same for long tail terms, therefore one bit is stole from DocFreq to check this case,
- * so that encoding of TotalTermFreq may be omitted.
- * </li>
- * </ul>
- *
- * @lucene.experimental
- */
-
-public class FSTOrdTermsWriter extends FieldsConsumer {
- static final String TERMS_INDEX_EXTENSION = "tix";
- static final String TERMS_BLOCK_EXTENSION = "tbk";
- static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT";
- public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CHECKSUM = 1;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
- public static final int SKIP_INTERVAL = 8;
-
- final PostingsWriterBase postingsWriter;
- final FieldInfos fieldInfos;
- final List<FieldMetaData> fields = new ArrayList<>();
- IndexOutput blockOut = null;
- IndexOutput indexOut = null;
-
- public FSTOrdTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) {
- final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
- final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_BLOCK_EXTENSION);
-
- this.postingsWriter = postingsWriter;
- this.fieldInfos = state.fieldInfos;
-
- bool success = false;
- try {
- this.indexOut = state.directory.createOutput(termsIndexFileName, state.context);
- this.blockOut = state.directory.createOutput(termsBlockFileName, state.context);
- writeHeader(indexOut);
- writeHeader(blockOut);
- this.postingsWriter.init(blockOut);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(indexOut, blockOut);
- }
- }
- }
-
- @Override
- public TermsConsumer addField(FieldInfo field) {
- return new TermsWriter(field);
- }
-
- @Override
- public void close() {
- if (blockOut != null) {
- IOException ioe = null;
- try {
- final long blockDirStart = blockOut.getFilePointer();
-
- // write field summary
- blockOut.writeVInt(fields.size());
- for (FieldMetaData field : fields) {
- blockOut.writeVInt(field.fieldInfo.number);
- blockOut.writeVLong(field.numTerms);
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- blockOut.writeVLong(field.sumTotalTermFreq);
- }
- blockOut.writeVLong(field.sumDocFreq);
- blockOut.writeVInt(field.docCount);
- blockOut.writeVInt(field.longsSize);
- blockOut.writeVLong(field.statsOut.getFilePointer());
- blockOut.writeVLong(field.metaLongsOut.getFilePointer());
- blockOut.writeVLong(field.metaBytesOut.getFilePointer());
-
- field.skipOut.writeTo(blockOut);
- field.statsOut.writeTo(blockOut);
- field.metaLongsOut.writeTo(blockOut);
- field.metaBytesOut.writeTo(blockOut);
- field.dict.save(indexOut);
- }
- writeTrailer(blockOut, blockDirStart);
- CodecUtil.writeFooter(indexOut);
- CodecUtil.writeFooter(blockOut);
- } catch (IOException ioe2) {
- ioe = ioe2;
- } finally {
- IOUtils.closeWhileHandlingException(ioe, blockOut, indexOut, postingsWriter);
- blockOut = null;
- }
- }
- }
-
- private void writeHeader(IndexOutput out) {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- }
- private void writeTrailer(IndexOutput out, long dirStart) {
- out.writeLong(dirStart);
- }
-
- private static class FieldMetaData {
- public FieldInfo fieldInfo;
- public long numTerms;
- public long sumTotalTermFreq;
- public long sumDocFreq;
- public int docCount;
- public int longsSize;
- public FST<Long> dict;
-
- // TODO: block encode each part
-
- // vint encode next skip point (fully decoded when reading)
- public RAMOutputStream skipOut;
- // vint encode df, (ttf-df)
- public RAMOutputStream statsOut;
- // vint encode monotonic long[] and length for corresponding byte[]
- public RAMOutputStream metaLongsOut;
- // generic byte[]
- public RAMOutputStream metaBytesOut;
- }
-
- final class TermsWriter extends TermsConsumer {
- private final Builder<Long> builder;
- private final PositiveIntOutputs outputs;
- private final FieldInfo fieldInfo;
- private final int longsSize;
- private long numTerms;
-
- private final IntsRef scratchTerm = new IntsRef();
- private final RAMOutputStream statsOut = new RAMOutputStream();
- private final RAMOutputStream metaLongsOut = new RAMOutputStream();
- private final RAMOutputStream metaBytesOut = new RAMOutputStream();
-
- private final RAMOutputStream skipOut = new RAMOutputStream();
- private long lastBlockStatsFP;
- private long lastBlockMetaLongsFP;
- private long lastBlockMetaBytesFP;
- private long[] lastBlockLongs;
-
- private long[] lastLongs;
- private long lastMetaBytesFP;
-
- TermsWriter(FieldInfo fieldInfo) {
- this.numTerms = 0;
- this.fieldInfo = fieldInfo;
- this.longsSize = postingsWriter.setField(fieldInfo);
- this.outputs = PositiveIntOutputs.getSingleton();
- this.builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
-
- this.lastBlockStatsFP = 0;
- this.lastBlockMetaLongsFP = 0;
- this.lastBlockMetaBytesFP = 0;
- this.lastBlockLongs = new long[longsSize];
-
- this.lastLongs = new long[longsSize];
- this.lastMetaBytesFP = 0;
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public PostingsConsumer startTerm(BytesRef text) {
- postingsWriter.startTerm();
- return postingsWriter;
- }
-
- @Override
- public void finishTerm(BytesRef text, TermStats stats) {
- if (numTerms > 0 && numTerms % SKIP_INTERVAL == 0) {
- bufferSkip();
- }
- // write term meta data into fst
- final long longs[] = new long[longsSize];
- final long delta = stats.totalTermFreq - stats.docFreq;
- if (stats.totalTermFreq > 0) {
- if (delta == 0) {
- statsOut.writeVInt(stats.docFreq<<1|1);
- } else {
- statsOut.writeVInt(stats.docFreq<<1|0);
- statsOut.writeVLong(stats.totalTermFreq-stats.docFreq);
- }
- } else {
- statsOut.writeVInt(stats.docFreq);
- }
- BlockTermState state = postingsWriter.newTermState();
- state.docFreq = stats.docFreq;
- state.totalTermFreq = stats.totalTermFreq;
- postingsWriter.finishTerm(state);
- postingsWriter.encodeTerm(longs, metaBytesOut, fieldInfo, state, true);
- for (int i = 0; i < longsSize; i++) {
- metaLongsOut.writeVLong(longs[i] - lastLongs[i]);
- lastLongs[i] = longs[i];
- }
- metaLongsOut.writeVLong(metaBytesOut.getFilePointer() - lastMetaBytesFP);
-
- builder.add(Util.toIntsRef(text, scratchTerm), numTerms);
- numTerms++;
-
- lastMetaBytesFP = metaBytesOut.getFilePointer();
- }
-
- @Override
- public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) {
- if (numTerms > 0) {
- final FieldMetaData metadata = new FieldMetaData();
- metadata.fieldInfo = fieldInfo;
- metadata.numTerms = numTerms;
- metadata.sumTotalTermFreq = sumTotalTermFreq;
- metadata.sumDocFreq = sumDocFreq;
- metadata.docCount = docCount;
- metadata.longsSize = longsSize;
- metadata.skipOut = skipOut;
- metadata.statsOut = statsOut;
- metadata.metaLongsOut = metaLongsOut;
- metadata.metaBytesOut = metaBytesOut;
- metadata.dict = builder.finish();
- fields.add(metadata);
- }
- }
-
- private void bufferSkip() {
- skipOut.writeVLong(statsOut.getFilePointer() - lastBlockStatsFP);
- skipOut.writeVLong(metaLongsOut.getFilePointer() - lastBlockMetaLongsFP);
- skipOut.writeVLong(metaBytesOut.getFilePointer() - lastBlockMetaBytesFP);
- for (int i = 0; i < longsSize; i++) {
- skipOut.writeVLong(lastLongs[i] - lastBlockLongs[i]);
- }
- lastBlockStatsFP = statsOut.getFilePointer();
- lastBlockMetaLongsFP = metaLongsOut.getFilePointer();
- lastBlockMetaBytesFP = metaBytesOut.getFilePointer();
- System.arraycopy(lastLongs, 0, lastBlockLongs, 0, longsSize);
- }
- }
-}
+using System;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using FieldInfos = org.apache.lucene.index.FieldInfos;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using DataOutput = org.apache.lucene.store.DataOutput;
+ using IndexOutput = org.apache.lucene.store.IndexOutput;
+ using RAMOutputStream = org.apache.lucene.store.RAMOutputStream;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using IntsRef = org.apache.lucene.util.IntsRef;
+ using Builder = org.apache.lucene.util.fst.Builder;
+ using FST = org.apache.lucene.util.fst.FST;
+ using PositiveIntOutputs = org.apache.lucene.util.fst.PositiveIntOutputs;
+ using Util = org.apache.lucene.util.fst.Util;
+
+ /// <summary>
+ /// FST-based term dict, using ord as FST output.
+ ///
+ /// The FST holds the mapping between <term, ord>, and
+ /// term's metadata is delta encoded into a single byte block.
+ ///
+ /// Typically the byte block consists of four parts:
+ /// 1. term statistics: docFreq, totalTermFreq;
+ /// 2. monotonic long[], e.g. the pointer to the postings list for that term;
+ /// 3. generic byte[], e.g. other information customized by postings base.
+ /// 4. single-level skip list to speed up metadata decoding by ord.
+ ///
+ /// <para>
+ /// Files:
+ /// <ul>
+ /// <li><tt>.tix</tt>: <a href="#Termindex">Term Index</a></li>
+ /// <li><tt>.tbk</tt>: <a href="#Termblock">Term Block</a></li>
+ /// </ul>
+ /// </para>
+ ///
+ /// <a name="Termindex" id="Termindex"></a>
+ /// <h3>Term Index</h3>
+ /// <para>
+ /// The .tix contains a list of FSTs, one for each field.
+ /// The FST maps a term to its corresponding order in current field.
+ /// </para>
+ ///
+ /// <ul>
+ /// <li>TermIndex(.tix) --> Header, TermFST<sup>NumFields</sup>, Footer</li>
+ /// <li>TermFST --> <seealso cref="FST FST<long>"/></li>
+ /// <li>Header --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
+ /// <li>Footer --> <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
+ /// </ul>
+ ///
+ /// <para>Notes:</para>
+ /// <ul>
+ /// <li>
+ /// Since terms are already sorted before writing to <a href="#Termblock">Term Block</a>,
+ /// their ords can directly used to seek term metadata from term block.
+ /// </li>
+ /// </ul>
+ ///
+ /// <a name="Termblock" id="Termblock"></a>
+ /// <h3>Term Block</h3>
+ /// <para>
+ /// The .tbk contains all the statistics and metadata for terms, along with field summary (e.g.
+ /// per-field data like number of documents in current field). For each field, there are four blocks:
+ /// <ul>
+ /// <li>statistics bytes block: contains term statistics; </li>
+ /// <li>metadata longs block: delta-encodes monotonic part of metadata; </li>
+ /// <li>metadata bytes block: encodes other parts of metadata; </li>
+ /// <li>skip block: contains skip data, to speed up metadata seeking and decoding</li>
+ /// </ul>
+ /// </para>
+ ///
+ /// <para>File Format:</para>
+ /// <ul>
+ /// <li>TermBlock(.tbk) --> Header, <i>PostingsHeader</i>, FieldSummary, DirOffset</li>
+ /// <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, SumTotalTermFreq?, SumDocFreq,
+ /// DocCount, LongsSize, DataBlock > <sup>NumFields</sup>, Footer</li>
+ ///
+ /// <li>DataBlock --> StatsBlockLength, MetaLongsBlockLength, MetaBytesBlockLength,
+ /// SkipBlock, StatsBlock, MetaLongsBlock, MetaBytesBlock </li>
+ /// <li>SkipBlock --> < StatsFPDelta, MetaLongsSkipFPDelta, MetaBytesSkipFPDelta,
+ /// MetaLongsSkipDelta<sup>LongsSize</sup> ><sup>NumTerms</sup>
+ /// <li>StatsBlock --> < DocFreq[Same?], (TotalTermFreq-DocFreq) ? > <sup>NumTerms</sup>
+ /// <li>MetaLongsBlock --> < LongDelta<sup>LongsSize</sup>, BytesSize > <sup>NumTerms</sup>
+ /// <li>MetaBytesBlock --> Byte <sup>MetaBytesBlockLength</sup>
+ /// <li>Header --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
+ /// <li>DirOffset --> <seealso cref="DataOutput#writeLong Uint64"/></li>
+ /// <li>NumFields, FieldNumber, DocCount, DocFreq, LongsSize,
+ /// FieldNumber, DocCount --> <seealso cref="DataOutput#writeVInt VInt"/></li>
+ /// <li>NumTerms, SumTotalTermFreq, SumDocFreq, StatsBlockLength, MetaLongsBlockLength, MetaBytesBlockLength,
+ /// StatsFPDelta, MetaLongsSkipFPDelta, MetaBytesSkipFPDelta, MetaLongsSkipStart, TotalTermFreq,
+ /// LongDelta,--> <seealso cref="DataOutput#writeVLong VLong"/></li>
+ /// <li>Footer --> <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
+ /// </ul>
+ /// <para>Notes: </para>
+ /// <ul>
+ /// <li>
+ /// The format of PostingsHeader and MetaBytes are customized by the specific postings implementation:
+ /// they contain arbitrary per-file data (such as parameters or versioning information), and per-term data
+ /// (non-monotonic ones like pulsed postings data).
+ /// </li>
+ /// <li>
+ /// During initialization the reader will load all the blocks into memory. SkipBlock will be decoded, so that during seek
+ /// term dict can lookup file pointers directly. StatsFPDelta, MetaLongsSkipFPDelta, etc. are file offset
+ /// for every SkipInterval's term. MetaLongsSkipDelta is the difference from previous one, which indicates
+ /// the value of preceding metadata longs for every SkipInterval's term.
+ /// </li>
+ /// <li>
+ /// DocFreq is the count of documents which contain the term. TotalTermFreq is the total number of occurrences of the term.
+ /// Usually these two values are the same for long tail terms, therefore one bit is stole from DocFreq to check this case,
+ /// so that encoding of TotalTermFreq may be omitted.
+ /// </li>
+ /// </ul>
+ ///
+ /// @lucene.experimental
+ /// </summary>
+
+ public class FSTOrdTermsWriter : FieldsConsumer
+ {
+ internal const string TERMS_INDEX_EXTENSION = "tix";
+ internal const string TERMS_BLOCK_EXTENSION = "tbk";
+ internal const string TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT";
+ public const int TERMS_VERSION_START = 0;
+ public const int TERMS_VERSION_CHECKSUM = 1;
+ public const int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
+ public const int SKIP_INTERVAL = 8;
+
+ internal readonly PostingsWriterBase postingsWriter;
+ internal readonly FieldInfos fieldInfos;
+ internal readonly IList<FieldMetaData> fields = new List<FieldMetaData>();
+ internal IndexOutput blockOut = null;
+ internal IndexOutput indexOut = null;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public FSTOrdTermsWriter(org.apache.lucene.index.SegmentWriteState state, org.apache.lucene.codecs.PostingsWriterBase postingsWriter) throws java.io.IOException
+ public FSTOrdTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String termsIndexFileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
+ string termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String termsBlockFileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_BLOCK_EXTENSION);
+ string termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_BLOCK_EXTENSION);
+
+ this.postingsWriter = postingsWriter;
+ this.fieldInfos = state.fieldInfos;
+
+ bool success = false;
+ try
+ {
+ this.indexOut = state.directory.createOutput(termsIndexFileName, state.context);
+ this.blockOut = state.directory.createOutput(termsBlockFileName, state.context);
+ writeHeader(indexOut);
+ writeHeader(blockOut);
+ this.postingsWriter.init(blockOut);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(indexOut, blockOut);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.TermsConsumer addField(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override TermsConsumer addField(FieldInfo field)
+ {
+ return new TermsWriter(this, field);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ if (blockOut != null)
+ {
+ IOException ioe = null;
+ try
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long blockDirStart = blockOut.getFilePointer();
+ long blockDirStart = blockOut.FilePointer;
+
+ // write field summary
+ blockOut.writeVInt(fields.Count);
+ foreach (FieldMetaData field in fields)
+ {
+ blockOut.writeVInt(field.fieldInfo.number);
+ blockOut.writeVLong(field.numTerms);
+ if (field.fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY)
+ {
+ blockOut.writeVLong(field.sumTotalTermFreq);
+ }
+ blockOut.writeVLong(field.sumDocFreq);
+ blockOut.writeVInt(field.docCount);
+ blockOut.writeVInt(field.longsSize);
+ blockOut.writeVLong(field.statsOut.FilePointer);
+ blockOut.writeVLong(field.metaLongsOut.FilePointer);
+ blockOut.writeVLong(field.metaBytesOut.FilePointer);
+
+ field.skipOut.writeTo(blockOut);
+ field.statsOut.writeTo(blockOut);
+ field.metaLongsOut.writeTo(blockOut);
+ field.metaBytesOut.writeTo(blockOut);
+ field.dict.save(indexOut);
+ }
+ writeTrailer(blockOut, blockDirStart);
+ CodecUtil.writeFooter(indexOut);
+ CodecUtil.writeFooter(blockOut);
+ }
+ catch (IOException ioe2)
+ {
+ ioe = ioe2;
+ }
+ finally
+ {
+ IOUtils.closeWhileHandlingException(ioe, blockOut, indexOut, postingsWriter);
+ blockOut = null;
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void writeHeader(org.apache.lucene.store.IndexOutput out) throws java.io.IOException
+ private void writeHeader(IndexOutput @out)
+ {
+ CodecUtil.writeHeader(@out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void writeTrailer(org.apache.lucene.store.IndexOutput out, long dirStart) throws java.io.IOException
+ private void writeTrailer(IndexOutput @out, long dirStart)
+ {
+ @out.writeLong(dirStart);
+ }
+
+ private class FieldMetaData
+ {
+ public FieldInfo fieldInfo;
+ public long numTerms;
+ public long sumTotalTermFreq;
+ public long sumDocFreq;
+ public int docCount;
+ public int longsSize;
+ public FST<long?> dict;
+
+ // TODO: block encode each part
+
+ // vint encode next skip point (fully decoded when reading)
+ public RAMOutputStream skipOut;
+ // vint encode df, (ttf-df)
+ public RAMOutputStream statsOut;
+ // vint encode monotonic long[] and length for corresponding byte[]
+ public RAMOutputStream metaLongsOut;
+ // generic byte[]
+ public RAMOutputStream metaBytesOut;
+ }
+
+ internal sealed class TermsWriter : TermsConsumer
+ {
+ private readonly FSTOrdTermsWriter outerInstance;
+
+ internal readonly Builder<long?> builder;
+ internal readonly PositiveIntOutputs outputs;
+ internal readonly FieldInfo fieldInfo;
+ internal readonly int longsSize;
+ internal long numTerms;
+
+ internal readonly IntsRef scratchTerm = new IntsRef();
+ internal readonly RAMOutputStream statsOut = new RAMOutputStream();
+ internal readonly RAMOutputStream metaLongsOut = new RAMOutputStream();
+ internal readonly RAMOutputStream metaBytesOut = new RAMOutputStream();
+
+ internal readonly RAMOutputStream skipOut = new RAMOutputStream();
+ internal long lastBlockStatsFP;
+ internal long lastBlockMetaLongsFP;
+ internal long lastBlockMetaBytesFP;
+ internal long[] lastBlockLongs;
+
+ internal long[] lastLongs;
+ internal long lastMetaBytesFP;
+
+ internal TermsWriter(FSTOrdTermsWriter outerInstance, FieldInfo fieldInfo)
+ {
+ this.outerInstance = outerInstance;
+ this.numTerms = 0;
+ this.fieldInfo = fieldInfo;
+ this.longsSize = outerInstance.postingsWriter.setField(fieldInfo);
+ this.outputs = PositiveIntOutputs.Singleton;
+ this.builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
+
+ this.lastBlockStatsFP = 0;
+ this.lastBlockMetaLongsFP = 0;
+ this.lastBlockMetaBytesFP = 0;
+ this.lastBlockLongs = new long[longsSize];
+
+ this.lastLongs = new long[longsSize];
+ this.lastMetaBytesFP = 0;
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.PostingsConsumer startTerm(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+ public override PostingsConsumer startTerm(BytesRef text)
+ {
+ outerInstance.postingsWriter.startTerm();
+ return outerInstance.postingsWriter;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void finishTerm(org.apache.lucene.util.BytesRef text, org.apache.lucene.codecs.TermStats stats) throws java.io.IOException
+ public override void finishTerm(BytesRef text, TermStats stats)
+ {
+ if (numTerms > 0 && numTerms % SKIP_INTERVAL == 0)
+ {
+ bufferSkip();
+ }
+ // write term meta data into fst
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long longs[] = new long[longsSize];
+ long[] longs = new long[longsSize];
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long delta = stats.totalTermFreq - stats.docFreq;
+ long delta = stats.totalTermFreq - stats.docFreq;
+ if (stats.totalTermFreq > 0)
+ {
+ if (delta == 0)
+ {
+ statsOut.writeVInt(stats.docFreq << 1 | 1);
+ }
+ else
+ {
+ statsOut.writeVInt(stats.docFreq << 1 | 0);
+ statsOut.writeVLong(stats.totalTermFreq - stats.docFreq);
+ }
+ }
+ else
+ {
+ statsOut.writeVInt(stats.docFreq);
+ }
+ BlockTermState state = outerInstance.postingsWriter.newTermState();
+ state.docFreq = stats.docFreq;
+ state.totalTermFreq = stats.totalTermFreq;
+ outerInstance.postingsWriter.finishTerm(state);
+ outerInstance.postingsWriter.encodeTerm(longs, metaBytesOut, fieldInfo, state, true);
+ for (int i = 0; i < longsSize; i++)
+ {
+ metaLongsOut.writeVLong(longs[i] - lastLongs[i]);
+ lastLongs[i] = longs[i];
+ }
+ metaLongsOut.writeVLong(metaBytesOut.FilePointer - lastMetaBytesFP);
+
+ builder.add(Util.toIntsRef(text, scratchTerm), numTerms);
+ numTerms++;
+
+ lastMetaBytesFP = metaBytesOut.FilePointer;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws java.io.IOException
+ public override void finish(long sumTotalTermFreq, long sumDocFreq, int docCount)
+ {
+ if (numTerms > 0)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final FieldMetaData metadata = new FieldMetaData();
+ FieldMetaData metadata = new FieldMetaData();
+ metadata.fieldInfo = fieldInfo;
+ metadata.numTerms = numTerms;
+ metadata.sumTotalTermFreq = sumTotalTermFreq;
+ metadata.sumDocFreq = sumDocFreq;
+ metadata.docCount = docCount;
+ metadata.longsSize = longsSize;
+ metadata.skipOut = skipOut;
+ metadata.statsOut = statsOut;
+ metadata.metaLongsOut = metaLongsOut;
+ metadata.metaBytesOut = metaBytesOut;
+ metadata.dict = builder.finish();
+ outerInstance.fields.Add(metadata);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void bufferSkip() throws java.io.IOException
+ internal void bufferSkip()
+ {
+ skipOut.writeVLong(statsOut.FilePointer - lastBlockStatsFP);
+ skipOut.writeVLong(metaLongsOut.FilePointer - lastBlockMetaLongsFP);
+ skipOut.writeVLong(metaBytesOut.FilePointer - lastBlockMetaBytesFP);
+ for (int i = 0; i < longsSize; i++)
+ {
+ skipOut.writeVLong(lastLongs[i] - lastBlockLongs[i]);
+ }
+ lastBlockStatsFP = statsOut.FilePointer;
+ lastBlockMetaLongsFP = metaLongsOut.FilePointer;
+ lastBlockMetaBytesFP = metaBytesOut.FilePointer;
+ Array.Copy(lastLongs, 0, lastBlockLongs, 0, longsSize);
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTPostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTPostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/FSTPostingsFormat.cs
index c5ea2f1..f41001b 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTPostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTPostingsFormat.cs
@@ -1,83 +1,88 @@
-package codecs.memory;
+namespace org.apache.lucene.codecs.memory
+{
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.io.IOException;
+ using Lucene41PostingsWriter = org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
+ using Lucene41PostingsReader = org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using IOUtils = org.apache.lucene.util.IOUtils;
-import codecs.FieldsConsumer;
-import codecs.FieldsProducer;
-import codecs.PostingsFormat;
-import codecs.PostingsReaderBase;
-import codecs.PostingsWriterBase;
-import codecs.lucene41.Lucene41PostingsWriter;
-import codecs.lucene41.Lucene41PostingsReader;
-import index.FieldInfo.IndexOptions;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import util.IOUtils;
+ /// <summary>
+ /// FST term dict + Lucene41PBF
+ /// </summary>
-/**
- * FST term dict + Lucene41PBF
- */
+ public sealed class FSTPostingsFormat : PostingsFormat
+ {
+ public FSTPostingsFormat() : base("FST41")
+ {
+ }
-public final class FSTPostingsFormat extends PostingsFormat {
- public FSTPostingsFormat() {
- super("FST41");
- }
+ public override string ToString()
+ {
+ return Name;
+ }
- @Override
- public String toString() {
- return getName();
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
- PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);
+ bool success = false;
+ try
+ {
+ FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(postingsWriter);
+ }
+ }
+ }
- bool success = false;
- try {
- FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override FieldsProducer fieldsProducer(SegmentReadState state)
+ {
+ PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ bool success = false;
+ try
+ {
+ FieldsProducer ret = new FSTTermsReader(state, postingsReader);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(postingsReader);
+ }
+ }
+ }
+ }
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) {
- PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- state.context,
- state.segmentSuffix);
- bool success = false;
- try {
- FieldsProducer ret = new FSTTermsReader(state, postingsReader);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsReader);
- }
- }
- }
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTPulsing41PostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTPulsing41PostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/FSTPulsing41PostingsFormat.cs
index 0db3cbd..7ff2a8c 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTPulsing41PostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTPulsing41PostingsFormat.cs
@@ -1,92 +1,102 @@
-package codecs.memory;
+namespace org.apache.lucene.codecs.memory
+{
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.io.IOException;
+ using Lucene41PostingsWriter = org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
+ using Lucene41PostingsReader = org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+ using Lucene41PostingsBaseFormat = org.apache.lucene.codecs.lucene41.Lucene41PostingsBaseFormat;
+ using Lucene41PostingsFormat = org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
+ using PulsingPostingsWriter = org.apache.lucene.codecs.pulsing.PulsingPostingsWriter;
+ using PulsingPostingsReader = org.apache.lucene.codecs.pulsing.PulsingPostingsReader;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using IOUtils = org.apache.lucene.util.IOUtils;
-import codecs.FieldsConsumer;
-import codecs.FieldsProducer;
-import codecs.PostingsBaseFormat;
-import codecs.PostingsFormat;
-import codecs.PostingsReaderBase;
-import codecs.PostingsWriterBase;
-import codecs.lucene41.Lucene41PostingsWriter;
-import codecs.lucene41.Lucene41PostingsReader;
-import codecs.lucene41.Lucene41PostingsBaseFormat;
-import codecs.lucene41.Lucene41PostingsFormat;
-import codecs.pulsing.PulsingPostingsWriter;
-import codecs.pulsing.PulsingPostingsReader;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import util.IOUtils;
+ /// <summary>
+ /// FST + Pulsing41, test only, since
+ /// FST does no delta encoding here!
+ /// @lucene.experimental
+ /// </summary>
-/** FST + Pulsing41, test only, since
- * FST does no delta encoding here!
- * @lucene.experimental */
+ public class FSTPulsing41PostingsFormat : PostingsFormat
+ {
+ private readonly PostingsBaseFormat wrappedPostingsBaseFormat;
+ private readonly int freqCutoff;
-public class FSTPulsing41PostingsFormat extends PostingsFormat {
- private final PostingsBaseFormat wrappedPostingsBaseFormat;
- private final int freqCutoff;
+ public FSTPulsing41PostingsFormat() : this(1)
+ {
+ }
- public FSTPulsing41PostingsFormat() {
- this(1);
- }
-
- public FSTPulsing41PostingsFormat(int freqCutoff) {
- super("FSTPulsing41");
- this.wrappedPostingsBaseFormat = new Lucene41PostingsBaseFormat();
- this.freqCutoff = freqCutoff;
- }
+ public FSTPulsing41PostingsFormat(int freqCutoff) : base("FSTPulsing41")
+ {
+ this.wrappedPostingsBaseFormat = new Lucene41PostingsBaseFormat();
+ this.freqCutoff = freqCutoff;
+ }
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
- PostingsWriterBase docsWriter = null;
- PostingsWriterBase pulsingWriter = null;
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ PostingsWriterBase docsWriter = null;
+ PostingsWriterBase pulsingWriter = null;
- bool success = false;
- try {
- docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state);
- pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter);
- FieldsConsumer ret = new FSTTermsWriter(state, pulsingWriter);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter);
- }
- }
- }
+ bool success = false;
+ try
+ {
+ docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state);
+ pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter);
+ FieldsConsumer ret = new FSTTermsWriter(state, pulsingWriter);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter);
+ }
+ }
+ }
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) {
- PostingsReaderBase docsReader = null;
- PostingsReaderBase pulsingReader = null;
- bool success = false;
- try {
- docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state);
- pulsingReader = new PulsingPostingsReader(state, docsReader);
- FieldsProducer ret = new FSTTermsReader(state, pulsingReader);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docsReader, pulsingReader);
- }
- }
- }
-}
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override FieldsProducer fieldsProducer(SegmentReadState state)
+ {
+ PostingsReaderBase docsReader = null;
+ PostingsReaderBase pulsingReader = null;
+ bool success = false;
+ try
+ {
+ docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state);
+ pulsingReader = new PulsingPostingsReader(state, docsReader);
+ FieldsProducer ret = new FSTTermsReader(state, pulsingReader);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(docsReader, pulsingReader);
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTTermOutputs.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermOutputs.cs b/src/Lucene.Net.Codecs/Memory/FSTTermOutputs.cs
index 55a2bd6..104ff61 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTTermOutputs.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTTermOutputs.cs
@@ -1,331 +1,423 @@
-package codecs.memory;
+using System.Diagnostics;
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+namespace org.apache.lucene.codecs.memory
+{
-import java.io.IOException;
-import java.util.Arrays;
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import index.FieldInfo;
-import index.FieldInfo.IndexOptions;
-import store.DataInput;
-import store.DataOutput;
-import util.fst.Outputs;
-import util.LongsRef;
-/**
- * An FST {@link Outputs} implementation for
- * {@link FSTTermsWriter}.
- *
- * @lucene.experimental
- */
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using DataInput = org.apache.lucene.store.DataInput;
+ using DataOutput = org.apache.lucene.store.DataOutput;
+ using Outputs = org.apache.lucene.util.fst.Outputs;
+ using LongsRef = org.apache.lucene.util.LongsRef;
-// NOTE: outputs should be per-field, since
-// longsSize is fixed for each field
-class FSTTermOutputs extends Outputs<FSTTermOutputs.TermData> {
- private final static TermData NO_OUTPUT = new TermData();
- //private static bool TEST = false;
- private final bool hasPos;
- private final int longsSize;
+ /// <summary>
+ /// An FST <seealso cref="Outputs"/> implementation for
+ /// <seealso cref="FSTTermsWriter"/>.
+ ///
+ /// @lucene.experimental
+ /// </summary>
- /**
- * Represents the metadata for one term.
- * On an FST, only long[] part is 'shared' and pushed towards root.
- * byte[] and term stats will be kept on deeper arcs.
- */
- static class TermData {
- long[] longs;
- byte[] bytes;
- int docFreq;
- long totalTermFreq;
- TermData() {
- this.longs = null;
- this.bytes = null;
- this.docFreq = 0;
- this.totalTermFreq = -1;
- }
- TermData(long[] longs, byte[] bytes, int docFreq, long totalTermFreq) {
- this.longs = longs;
- this.bytes = bytes;
- this.docFreq = docFreq;
- this.totalTermFreq = totalTermFreq;
- }
+ // NOTE: outputs should be per-field, since
+ // longsSize is fixed for each field
+ internal class FSTTermOutputs : Outputs<FSTTermOutputs.TermData>
+ {
+ private static readonly TermData NO_OUTPUT = new TermData();
+ //private static boolean TEST = false;
+ private readonly bool hasPos;
+ private readonly int longsSize;
- // NOTE: actually, FST nodes are seldom
- // identical when outputs on their arcs
- // aren't NO_OUTPUTs.
- @Override
- public int hashCode() {
- int hash = 0;
- if (longs != null) {
- final int end = longs.length;
- for (int i = 0; i < end; i++) {
- hash -= longs[i];
- }
- }
- if (bytes != null) {
- hash = -hash;
- final int end = bytes.length;
- for (int i = 0; i < end; i++) {
- hash += bytes[i];
- }
- }
- hash += docFreq + totalTermFreq;
- return hash;
- }
+ /// <summary>
+ /// Represents the metadata for one term.
+ /// On an FST, only long[] part is 'shared' and pushed towards root.
+ /// byte[] and term stats will be kept on deeper arcs.
+ /// </summary>
+ internal class TermData
+ {
+ internal long[] longs;
+ internal sbyte[] bytes;
+ internal int docFreq;
+ internal long totalTermFreq;
+ internal TermData()
+ {
+ this.longs = null;
+ this.bytes = null;
+ this.docFreq = 0;
+ this.totalTermFreq = -1;
+ }
+ internal TermData(long[] longs, sbyte[] bytes, int docFreq, long totalTermFreq)
+ {
+ this.longs = longs;
+ this.bytes = bytes;
+ this.docFreq = docFreq;
+ this.totalTermFreq = totalTermFreq;
+ }
- @Override
- public bool equals(Object other_) {
- if (other_ == this) {
- return true;
- } else if (!(other_ instanceof FSTTermOutputs.TermData)) {
- return false;
- }
- TermData other = (TermData) other_;
- return statsEqual(this, other) &&
- longsEqual(this, other) &&
- bytesEqual(this, other);
+ // NOTE: actually, FST nodes are seldom
+ // identical when outputs on their arcs
+ // aren't NO_OUTPUTs.
+ public override int GetHashCode()
+ {
+ int hash = 0;
+ if (longs != null)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int end = longs.length;
+ int end = longs.Length;
+ for (int i = 0; i < end; i++)
+ {
+ hash -= (int)longs[i];
+ }
+ }
+ if (bytes != null)
+ {
+ hash = -hash;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int end = bytes.length;
+ int end = bytes.Length;
+ for (int i = 0; i < end; i++)
+ {
+ hash += bytes[i];
+ }
+ }
+ hash += (int)(docFreq + totalTermFreq);
+ return hash;
+ }
- }
- }
-
- protected FSTTermOutputs(FieldInfo fieldInfo, int longsSize) {
- this.hasPos = (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY);
- this.longsSize = longsSize;
- }
+ public override bool Equals(object other_)
+ {
+ if (other_ == this)
+ {
+ return true;
+ }
+ else if (!(other_ is FSTTermOutputs.TermData))
+ {
+ return false;
+ }
+ TermData other = (TermData) other_;
+ return statsEqual(this, other) && longsEqual(this, other) && bytesEqual(this, other);
- @Override
- //
- // The return value will be the smaller one, when these two are
- // 'comparable', i.e.
- // 1. every value in t1 is not larger than in t2, or
- // 2. every value in t1 is not smaller than t2.
- //
- public TermData common(TermData t1, TermData t2) {
- //if (TEST) System.out.print("common("+t1+", "+t2+") = ");
- if (t1 == NO_OUTPUT || t2 == NO_OUTPUT) {
- //if (TEST) System.out.println("ret:"+NO_OUTPUT);
- return NO_OUTPUT;
- }
- Debug.Assert( t1.longs.length == t2.longs.length;
+ }
+ }
- long[] min = t1.longs, max = t2.longs;
- int pos = 0;
- TermData ret;
+ protected internal FSTTermOutputs(FieldInfo fieldInfo, int longsSize)
+ {
+ this.hasPos = (fieldInfo.IndexOptions != FieldInfo.IndexOptions.DOCS_ONLY);
+ this.longsSize = longsSize;
+ }
- while (pos < longsSize && min[pos] == max[pos]) {
- pos++;
- }
- if (pos < longsSize) { // unequal long[]
- if (min[pos] > max[pos]) {
- min = t2.longs;
- max = t1.longs;
- }
- // check whether strictly smaller
- while (pos < longsSize && min[pos] <= max[pos]) {
- pos++;
- }
- if (pos < longsSize || allZero(min)) { // not comparable or all-zero
- ret = NO_OUTPUT;
- } else {
- ret = new TermData(min, null, 0, -1);
- }
- } else { // equal long[]
- if (statsEqual(t1, t2) && bytesEqual(t1, t2)) {
- ret = t1;
- } else if (allZero(min)) {
- ret = NO_OUTPUT;
- } else {
- ret = new TermData(min, null, 0, -1);
- }
- }
- //if (TEST) System.out.println("ret:"+ret);
- return ret;
- }
+ public override TermData common(TermData t1, TermData t2)
+ //
+ // The return value will be the smaller one, when these two are
+ // 'comparable', i.e.
+ // 1. every value in t1 is not larger than in t2, or
+ // 2. every value in t1 is not smaller than t2.
+ //
+ {
+ //if (TEST) System.out.print("common("+t1+", "+t2+") = ");
+ if (t1 == NO_OUTPUT || t2 == NO_OUTPUT)
+ {
+ //if (TEST) System.out.println("ret:"+NO_OUTPUT);
+ return NO_OUTPUT;
+ }
+ Debug.Assert(t1.longs.Length == t2.longs.Length);
- @Override
- public TermData subtract(TermData t1, TermData t2) {
- //if (TEST) System.out.print("subtract("+t1+", "+t2+") = ");
- if (t2 == NO_OUTPUT) {
- //if (TEST) System.out.println("ret:"+t1);
- return t1;
- }
- Debug.Assert( t1.longs.length == t2.longs.length;
+ long[] min = t1.longs, max = t2.longs;
+ int pos = 0;
+ TermData ret;
- int pos = 0;
- long diff = 0;
- long[] share = new long[longsSize];
+ while (pos < longsSize && min[pos] == max[pos])
+ {
+ pos++;
+ }
+ if (pos < longsSize) // unequal long[]
+ {
+ if (min[pos] > max[pos])
+ {
+ min = t2.longs;
+ max = t1.longs;
+ }
+ // check whether strictly smaller
+ while (pos < longsSize && min[pos] <= max[pos])
+ {
+ pos++;
+ }
+ if (pos < longsSize || allZero(min)) // not comparable or all-zero
+ {
+ ret = NO_OUTPUT;
+ }
+ else
+ {
+ ret = new TermData(min, null, 0, -1);
+ }
+ } // equal long[]
+ else
+ {
+ if (statsEqual(t1, t2) && bytesEqual(t1, t2))
+ {
+ ret = t1;
+ }
+ else if (allZero(min))
+ {
+ ret = NO_OUTPUT;
+ }
+ else
+ {
+ ret = new TermData(min, null, 0, -1);
+ }
+ }
+ //if (TEST) System.out.println("ret:"+ret);
+ return ret;
+ }
- while (pos < longsSize) {
- share[pos] = t1.longs[pos] - t2.longs[pos];
- diff += share[pos];
- pos++;
- }
+ public override TermData subtract(TermData t1, TermData t2)
+ {
+ //if (TEST) System.out.print("subtract("+t1+", "+t2+") = ");
+ if (t2 == NO_OUTPUT)
+ {
+ //if (TEST) System.out.println("ret:"+t1);
+ return t1;
+ }
+ Debug.Assert(t1.longs.Length == t2.longs.Length);
- TermData ret;
- if (diff == 0 && statsEqual(t1, t2) && bytesEqual(t1, t2)) {
- ret = NO_OUTPUT;
- } else {
- ret = new TermData(share, t1.bytes, t1.docFreq, t1.totalTermFreq);
- }
- //if (TEST) System.out.println("ret:"+ret);
- return ret;
- }
+ int pos = 0;
+ long diff = 0;
+ long[] share = new long[longsSize];
- // TODO: if we refactor a 'addSelf(TermData other)',
- // we can gain about 5~7% for fuzzy queries, however this also
- // means we are putting too much stress on FST Outputs decoding?
- @Override
- public TermData add(TermData t1, TermData t2) {
- //if (TEST) System.out.print("add("+t1+", "+t2+") = ");
- if (t1 == NO_OUTPUT) {
- //if (TEST) System.out.println("ret:"+t2);
- return t2;
- } else if (t2 == NO_OUTPUT) {
- //if (TEST) System.out.println("ret:"+t1);
- return t1;
- }
- Debug.Assert( t1.longs.length == t2.longs.length;
+ while (pos < longsSize)
+ {
+ share[pos] = t1.longs[pos] - t2.longs[pos];
+ diff += share[pos];
+ pos++;
+ }
- int pos = 0;
- long[] accum = new long[longsSize];
+ TermData ret;
+ if (diff == 0 && statsEqual(t1, t2) && bytesEqual(t1, t2))
+ {
+ ret = NO_OUTPUT;
+ }
+ else
+ {
+ ret = new TermData(share, t1.bytes, t1.docFreq, t1.totalTermFreq);
+ }
+ //if (TEST) System.out.println("ret:"+ret);
+ return ret;
+ }
- while (pos < longsSize) {
- accum[pos] = t1.longs[pos] + t2.longs[pos];
- pos++;
- }
+ // TODO: if we refactor a 'addSelf(TermData other)',
+ // we can gain about 5~7% for fuzzy queries, however this also
+ // means we are putting too much stress on FST Outputs decoding?
+ public override TermData add(TermData t1, TermData t2)
+ {
+ //if (TEST) System.out.print("add("+t1+", "+t2+") = ");
+ if (t1 == NO_OUTPUT)
+ {
+ //if (TEST) System.out.println("ret:"+t2);
+ return t2;
+ }
+ else if (t2 == NO_OUTPUT)
+ {
+ //if (TEST) System.out.println("ret:"+t1);
+ return t1;
+ }
+ Debug.Assert(t1.longs.Length == t2.longs.Length);
- TermData ret;
- if (t2.bytes != null || t2.docFreq > 0) {
- ret = new TermData(accum, t2.bytes, t2.docFreq, t2.totalTermFreq);
- } else {
- ret = new TermData(accum, t1.bytes, t1.docFreq, t1.totalTermFreq);
- }
- //if (TEST) System.out.println("ret:"+ret);
- return ret;
- }
+ int pos = 0;
+ long[] accum = new long[longsSize];
- @Override
- public void write(TermData data, DataOutput out) {
- int bit0 = allZero(data.longs) ? 0 : 1;
- int bit1 = ((data.bytes == null || data.bytes.length == 0) ? 0 : 1) << 1;
- int bit2 = ((data.docFreq == 0) ? 0 : 1) << 2;
- int bits = bit0 | bit1 | bit2;
- if (bit1 > 0) { // determine extra length
- if (data.bytes.length < 32) {
- bits |= (data.bytes.length << 3);
- out.writeByte((byte)bits);
- } else {
- out.writeByte((byte)bits);
- out.writeVInt(data.bytes.length);
- }
- } else {
- out.writeByte((byte)bits);
- }
- if (bit0 > 0) { // not all-zero case
- for (int pos = 0; pos < longsSize; pos++) {
- out.writeVLong(data.longs[pos]);
- }
- }
- if (bit1 > 0) { // bytes exists
- out.writeBytes(data.bytes, 0, data.bytes.length);
- }
- if (bit2 > 0) { // stats exist
- if (hasPos) {
- if (data.docFreq == data.totalTermFreq) {
- out.writeVInt((data.docFreq << 1) | 1);
- } else {
- out.writeVInt((data.docFreq << 1));
- out.writeVLong(data.totalTermFreq - data.docFreq);
- }
- } else {
- out.writeVInt(data.docFreq);
- }
- }
- }
+ while (pos < longsSize)
+ {
+ accum[pos] = t1.longs[pos] + t2.longs[pos];
+ pos++;
+ }
- @Override
- public TermData read(DataInput in) {
- long[] longs = new long[longsSize];
- byte[] bytes = null;
- int docFreq = 0;
- long totalTermFreq = -1;
- int bits = in.readByte() & 0xff;
- int bit0 = bits & 1;
- int bit1 = bits & 2;
- int bit2 = bits & 4;
- int bytesSize = (bits >>> 3);
- if (bit1 > 0 && bytesSize == 0) { // determine extra length
- bytesSize = in.readVInt();
- }
- if (bit0 > 0) { // not all-zero case
- for (int pos = 0; pos < longsSize; pos++) {
- longs[pos] = in.readVLong();
- }
- }
- if (bit1 > 0) { // bytes exists
- bytes = new byte[bytesSize];
- in.readBytes(bytes, 0, bytesSize);
- }
- if (bit2 > 0) { // stats exist
- int code = in.readVInt();
- if (hasPos) {
- totalTermFreq = docFreq = code >>> 1;
- if ((code & 1) == 0) {
- totalTermFreq += in.readVLong();
- }
- } else {
- docFreq = code;
- }
- }
- return new TermData(longs, bytes, docFreq, totalTermFreq);
- }
+ TermData ret;
+ if (t2.bytes != null || t2.docFreq > 0)
+ {
+ ret = new TermData(accum, t2.bytes, t2.docFreq, t2.totalTermFreq);
+ }
+ else
+ {
+ ret = new TermData(accum, t1.bytes, t1.docFreq, t1.totalTermFreq);
+ }
+ //if (TEST) System.out.println("ret:"+ret);
+ return ret;
+ }
- @Override
- public TermData getNoOutput() {
- return NO_OUTPUT;
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void write(TermData data, org.apache.lucene.store.DataOutput out) throws java.io.IOException
+ public override void write(TermData data, DataOutput @out)
+ {
+ int bit0 = allZero(data.longs) ? 0 : 1;
+ int bit1 = ((data.bytes == null || data.bytes.Length == 0) ? 0 : 1) << 1;
+ int bit2 = ((data.docFreq == 0) ? 0 : 1) << 2;
+ int bits = bit0 | bit1 | bit2;
+ if (bit1 > 0) // determine extra length
+ {
+ if (data.bytes.Length < 32)
+ {
+ bits |= (data.bytes.Length << 3);
+ @out.writeByte((sbyte)bits);
+ }
+ else
+ {
+ @out.writeByte((sbyte)bits);
+ @out.writeVInt(data.bytes.Length);
+ }
+ }
+ else
+ {
+ @out.writeByte((sbyte)bits);
+ }
+ if (bit0 > 0) // not all-zero case
+ {
+ for (int pos = 0; pos < longsSize; pos++)
+ {
+ @out.writeVLong(data.longs[pos]);
+ }
+ }
+ if (bit1 > 0) // bytes exists
+ {
+ @out.writeBytes(data.bytes, 0, data.bytes.Length);
+ }
+ if (bit2 > 0) // stats exist
+ {
+ if (hasPos)
+ {
+ if (data.docFreq == data.totalTermFreq)
+ {
+ @out.writeVInt((data.docFreq << 1) | 1);
+ }
+ else
+ {
+ @out.writeVInt((data.docFreq << 1));
+ @out.writeVLong(data.totalTermFreq - data.docFreq);
+ }
+ }
+ else
+ {
+ @out.writeVInt(data.docFreq);
+ }
+ }
+ }
- @Override
- public String outputToString(TermData data) {
- return data.toString();
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public TermData read(org.apache.lucene.store.DataInput in) throws java.io.IOException
+ public override TermData read(DataInput @in)
+ {
+ long[] longs = new long[longsSize];
+ sbyte[] bytes = null;
+ int docFreq = 0;
+ long totalTermFreq = -1;
+ int bits = @in.readByte() & 0xff;
+ int bit0 = bits & 1;
+ int bit1 = bits & 2;
+ int bit2 = bits & 4;
+ int bytesSize = ((int)((uint)bits >> 3));
+ if (bit1 > 0 && bytesSize == 0) // determine extra length
+ {
+ bytesSize = @in.readVInt();
+ }
+ if (bit0 > 0) // not all-zero case
+ {
+ for (int pos = 0; pos < longsSize; pos++)
+ {
+ longs[pos] = @in.readVLong();
+ }
+ }
+ if (bit1 > 0) // bytes exists
+ {
+ bytes = new sbyte[bytesSize];
+ @in.readBytes(bytes, 0, bytesSize);
+ }
+ if (bit2 > 0) // stats exist
+ {
+ int code = @in.readVInt();
+ if (hasPos)
+ {
+ totalTermFreq = docFreq = (int)((uint)code >> 1);
+ if ((code & 1) == 0)
+ {
+ totalTermFreq += @in.readVLong();
+ }
+ }
+ else
+ {
+ docFreq = code;
+ }
+ }
+ return new TermData(longs, bytes, docFreq, totalTermFreq);
+ }
- static bool statsEqual(final TermData t1, final TermData t2) {
- return t1.docFreq == t2.docFreq && t1.totalTermFreq == t2.totalTermFreq;
- }
- static bool bytesEqual(final TermData t1, final TermData t2) {
- if (t1.bytes == null && t2.bytes == null) {
- return true;
- }
- return t1.bytes != null && t2.bytes != null && Arrays.equals(t1.bytes, t2.bytes);
- }
- static bool longsEqual(final TermData t1, final TermData t2) {
- if (t1.longs == null && t2.longs == null) {
- return true;
- }
- return t1.longs != null && t2.longs != null && Arrays.equals(t1.longs, t2.longs);
- }
- static bool allZero(final long[] l) {
- for (int i = 0; i < l.length; i++) {
- if (l[i] != 0) {
- return false;
- }
- }
- return true;
- }
-}
+ public override TermData NoOutput
+ {
+ get
+ {
+ return NO_OUTPUT;
+ }
+ }
+
+ public override string outputToString(TermData data)
+ {
+ return data.ToString();
+ }
+
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+//ORIGINAL LINE: static boolean statsEqual(final TermData t1, final TermData t2)
+ internal static bool statsEqual(TermData t1, TermData t2)
+ {
+ return t1.docFreq == t2.docFreq && t1.totalTermFreq == t2.totalTermFreq;
+ }
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+//ORIGINAL LINE: static boolean bytesEqual(final TermData t1, final TermData t2)
+ internal static bool bytesEqual(TermData t1, TermData t2)
+ {
+ if (t1.bytes == null && t2.bytes == null)
+ {
+ return true;
+ }
+ return t1.bytes != null && t2.bytes != null && Arrays.Equals(t1.bytes, t2.bytes);
+ }
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+//ORIGINAL LINE: static boolean longsEqual(final TermData t1, final TermData t2)
+ internal static bool longsEqual(TermData t1, TermData t2)
+ {
+ if (t1.longs == null && t2.longs == null)
+ {
+ return true;
+ }
+ return t1.longs != null && t2.longs != null && Arrays.Equals(t1.longs, t2.longs);
+ }
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+//ORIGINAL LINE: static boolean allZero(final long[] l)
+ internal static bool allZero(long[] l)
+ {
+ for (int i = 0; i < l.Length; i++)
+ {
+ if (l[i] != 0)
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+}
\ No newline at end of file
[02/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/SepPostingsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/SepPostingsReader.cs b/src/Lucene.Net.Codecs/Sep/SepPostingsReader.cs
index e7b09e4..cd8c2d6 100644
--- a/src/Lucene.Net.Codecs/Sep/SepPostingsReader.cs
+++ b/src/Lucene.Net.Codecs/Sep/SepPostingsReader.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,698 +15,762 @@ package codecs.sep;
* limitations under the License.
*/
-import java.io.IOException;
-
-import codecs.BlockTermState;
-import codecs.CodecUtil;
-import codecs.PostingsReaderBase;
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.SegmentInfo;
-import index.TermState;
-import store.ByteArrayDataInput;
-import store.DataInput;
-import store.Directory;
-import store.IOContext;
-import store.IndexInput;
-import util.ArrayUtil;
-import util.Bits;
-import util.BytesRef;
-import util.IOUtils;
-
-/** Concrete class that reads the current doc/freq/skip
- * postings format.
- *
- * @lucene.experimental
- */
-
-// TODO: -- should we switch "hasProx" higher up? and
-// create two separate docs readers, one that also reads
-// prox and one that doesn't?
-
-public class SepPostingsReader extends PostingsReaderBase {
-
- final IntIndexInput freqIn;
- final IntIndexInput docIn;
- final IntIndexInput posIn;
- final IndexInput payloadIn;
- final IndexInput skipIn;
-
- int skipInterval;
- int maxSkipLevels;
- int skipMinimum;
-
- public SepPostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, String segmentSuffix) {
- bool success = false;
- try {
-
- final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION);
- docIn = intFactory.openInput(dir, docFileName, context);
-
- skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION), context);
-
- if (fieldInfos.hasFreq()) {
- freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION), context);
- } else {
- freqIn = null;
- }
- if (fieldInfos.hasProx()) {
- posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION), context);
- payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION), context);
- } else {
- posIn = null;
- payloadIn = null;
- }
- success = true;
- } finally {
- if (!success) {
- close();
- }
- }
- }
-
- @Override
- public void init(IndexInput termsIn) {
- // Make sure we are talking to the matching past writer
- CodecUtil.checkHeader(termsIn, SepPostingsWriter.CODEC,
- SepPostingsWriter.VERSION_START, SepPostingsWriter.VERSION_START);
- skipInterval = termsIn.readInt();
- maxSkipLevels = termsIn.readInt();
- skipMinimum = termsIn.readInt();
- }
-
- @Override
- public void close() {
- IOUtils.close(freqIn, docIn, skipIn, posIn, payloadIn);
- }
-
- private static final class SepTermState extends BlockTermState {
- // We store only the seek point to the docs file because
- // the rest of the info (freqIndex, posIndex, etc.) is
- // stored in the docs file:
- IntIndexInput.Index docIndex;
- IntIndexInput.Index posIndex;
- IntIndexInput.Index freqIndex;
- long payloadFP;
- long skipFP;
-
- @Override
- public SepTermState clone() {
- SepTermState other = new SepTermState();
- other.copyFrom(this);
- return other;
- }
-
- @Override
- public void copyFrom(TermState _other) {
- super.copyFrom(_other);
- SepTermState other = (SepTermState) _other;
- if (docIndex == null) {
- docIndex = other.docIndex.clone();
- } else {
- docIndex.copyFrom(other.docIndex);
- }
- if (other.freqIndex != null) {
- if (freqIndex == null) {
- freqIndex = other.freqIndex.clone();
- } else {
- freqIndex.copyFrom(other.freqIndex);
+namespace Lucene.Net.Codecs.Sep
+{
+
+ using System.Diagnostics;
+ using Index;
+ using Store;
+ using Util;
+
+ /// <summary>
+ /// Concrete class that reads the current doc/freq/skip
+ /// postings format.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ /// <remarks>
+ /// TODO: -- should we switch "hasProx" higher up? and
+ /// create two separate docs readers, one that also reads
+ /// prox and one that doesn't?
+ /// </remarks>
+ public class SepPostingsReader : PostingsReaderBase
+ {
+ private readonly IntIndexInput _freqIn;
+ private readonly IntIndexInput _docIn;
+ private readonly IntIndexInput _posIn;
+ private readonly IndexInput _payloadIn;
+ private readonly IndexInput _skipIn;
+
+ private int _skipInterval;
+ private int _maxSkipLevels;
+ private int _skipMinimum;
+
+ public SepPostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext context,
+ IntStreamFactory intFactory, string segmentSuffix)
+ {
+ var success = false;
+ try
+ {
+
+ var docFileName = IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix,
+ SepPostingsWriter.DOC_EXTENSION);
+ _docIn = intFactory.OpenInput(dir, docFileName, context);
+
+ _skipIn =
+ dir.OpenInput(
+ IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION),
+ context);
+
+ if (fieldInfos.HasFreq())
+ {
+ _freqIn = intFactory.OpenInput(dir,
+ IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION),
+ context);
+ }
+ else
+ {
+ _freqIn = null;
+ }
+ if (fieldInfos.HasProx())
+ {
+ _posIn = intFactory.OpenInput(dir,
+ IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, SepPostingsWriter.POS_EXTENSION),
+ context);
+ _payloadIn =
+ dir.OpenInput(
+ IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix,
+ SepPostingsWriter.PAYLOAD_EXTENSION), context);
+ }
+ else
+ {
+ _posIn = null;
+ _payloadIn = null;
+ }
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ Dispose();
+ }
+ }
}
- } else {
- freqIndex = null;
- }
- if (other.posIndex != null) {
- if (posIndex == null) {
- posIndex = other.posIndex.clone();
- } else {
- posIndex.copyFrom(other.posIndex);
- }
- } else {
- posIndex = null;
- }
- payloadFP = other.payloadFP;
- skipFP = other.skipFP;
- }
-
- @Override
- public String toString() {
- return super.toString() + " docIndex=" + docIndex + " freqIndex=" + freqIndex + " posIndex=" + posIndex + " payloadFP=" + payloadFP + " skipFP=" + skipFP;
- }
- }
-
- @Override
- public BlockTermState newTermState() {
- final SepTermState state = new SepTermState();
- state.docIndex = docIn.index();
- if (freqIn != null) {
- state.freqIndex = freqIn.index();
- }
- if (posIn != null) {
- state.posIndex = posIn.index();
- }
- return state;
- }
-
- @Override
- public void decodeTerm(long[] empty, DataInput in, FieldInfo fieldInfo, BlockTermState _termState, bool absolute)
- {
- final SepTermState termState = (SepTermState) _termState;
- termState.docIndex.read(in, absolute);
- if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- termState.freqIndex.read(in, absolute);
- if (fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- //System.out.println(" freqIndex=" + termState.freqIndex);
- termState.posIndex.read(in, absolute);
- //System.out.println(" posIndex=" + termState.posIndex);
- if (fieldInfo.hasPayloads()) {
- if (absolute) {
- termState.payloadFP = in.readVLong();
- } else {
- termState.payloadFP += in.readVLong();
- }
- //System.out.println(" payloadFP=" + termState.payloadFP);
- }
- }
- }
-
- if (termState.docFreq >= skipMinimum) {
- //System.out.println(" readSkip @ " + in.getPosition());
- if (absolute) {
- termState.skipFP = in.readVLong();
- } else {
- termState.skipFP += in.readVLong();
- }
- //System.out.println(" skipFP=" + termState.skipFP);
- } else if (absolute) {
- termState.skipFP = 0;
- }
- }
-
- @Override
- public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, int flags) {
- final SepTermState termState = (SepTermState) _termState;
- SepDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof SepDocsEnum)) {
- docsEnum = new SepDocsEnum();
- } else {
- docsEnum = (SepDocsEnum) reuse;
- if (docsEnum.startDocIn != docIn) {
- // If you are using ParellelReader, and pass in a
- // reused DocsAndPositionsEnum, it could have come
- // from another reader also using sep codec
- docsEnum = new SepDocsEnum();
- }
- }
-
- return docsEnum.init(fieldInfo, termState, liveDocs);
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs,
- DocsAndPositionsEnum reuse, int flags)
- {
-
- Debug.Assert( fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
- final SepTermState termState = (SepTermState) _termState;
- SepDocsAndPositionsEnum postingsEnum;
- if (reuse == null || !(reuse instanceof SepDocsAndPositionsEnum)) {
- postingsEnum = new SepDocsAndPositionsEnum();
- } else {
- postingsEnum = (SepDocsAndPositionsEnum) reuse;
- if (postingsEnum.startDocIn != docIn) {
- // If you are using ParellelReader, and pass in a
- // reused DocsAndPositionsEnum, it could have come
- // from another reader also using sep codec
- postingsEnum = new SepDocsAndPositionsEnum();
- }
- }
-
- return postingsEnum.init(fieldInfo, termState, liveDocs);
- }
-
- class SepDocsEnum extends DocsEnum {
- int docFreq;
- int doc = -1;
- int accum;
- int count;
- int freq;
- long freqStart;
-
- // TODO: -- should we do omitTF with 2 different enum classes?
- private bool omitTF;
- private IndexOptions indexOptions;
- private bool storePayloads;
- private Bits liveDocs;
- private final IntIndexInput.Reader docReader;
- private final IntIndexInput.Reader freqReader;
- private long skipFP;
-
- private final IntIndexInput.Index docIndex;
- private final IntIndexInput.Index freqIndex;
- private final IntIndexInput.Index posIndex;
- private final IntIndexInput startDocIn;
-
- // TODO: -- should we do hasProx with 2 different enum classes?
-
- bool skipped;
- SepSkipListReader skipper;
-
- SepDocsEnum() {
- startDocIn = docIn;
- docReader = docIn.reader();
- docIndex = docIn.index();
- if (freqIn != null) {
- freqReader = freqIn.reader();
- freqIndex = freqIn.index();
- } else {
- freqReader = null;
- freqIndex = null;
- }
- if (posIn != null) {
- posIndex = posIn.index(); // only init this so skipper can read it
- } else {
- posIndex = null;
- }
- }
- SepDocsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) {
- this.liveDocs = liveDocs;
- this.indexOptions = fieldInfo.getIndexOptions();
- omitTF = indexOptions == IndexOptions.DOCS_ONLY;
- storePayloads = fieldInfo.hasPayloads();
-
- // TODO: can't we only do this if consumer
- // skipped consuming the previous docs?
- docIndex.copyFrom(termState.docIndex);
- docIndex.seek(docReader);
-
- if (!omitTF) {
- freqIndex.copyFrom(termState.freqIndex);
- freqIndex.seek(freqReader);
- }
-
- docFreq = termState.docFreq;
- // NOTE: unused if docFreq < skipMinimum:
- skipFP = termState.skipFP;
- count = 0;
- doc = -1;
- accum = 0;
- freq = 1;
- skipped = false;
-
- return this;
- }
-
- @Override
- public int nextDoc() {
-
- while(true) {
- if (count == docFreq) {
- return doc = NO_MORE_DOCS;
+ public override void Init(IndexInput termsIn)
+ {
+ // Make sure we are talking to the matching past writer
+ CodecUtil.CheckHeader(termsIn, SepPostingsWriter.CODEC, SepPostingsWriter.VERSION_START,
+ SepPostingsWriter.VERSION_START);
+ _skipInterval = termsIn.ReadInt();
+ _maxSkipLevels = termsIn.ReadInt();
+ _skipMinimum = termsIn.ReadInt();
}
- count++;
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing) return;
- // Decode next doc
- //System.out.println("decode docDelta:");
- accum += docReader.next();
-
- if (!omitTF) {
- //System.out.println("decode freq:");
- freq = freqReader.next();
+ IOUtils.Close(_freqIn, _docIn, _skipIn, _posIn, _payloadIn);
}
- if (liveDocs == null || liveDocs.get(accum)) {
- break;
- }
- }
- return (doc = accum);
- }
-
- @Override
- public int freq() {
- return freq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public int advance(int target) {
+ public override BlockTermState NewTermState()
+ {
+ var state = new SepTermState {DOC_INDEX = _docIn.Index()};
- if ((target - skipInterval) >= doc && docFreq >= skipMinimum) {
+ if (_freqIn != null)
+ state.FREQ_INDEX = _freqIn.Index();
- // There are enough docs in the posting to have
- // skip data, and its not too close
-
- if (skipper == null) {
- // This DocsEnum has never done any skipping
- skipper = new SepSkipListReader(skipIn.clone(),
- freqIn,
- docIn,
- posIn,
- maxSkipLevels, skipInterval);
+ if (_posIn != null)
+ state.POS_INDEX = _posIn.Index();
+ return state;
}
- if (!skipped) {
- // We haven't yet skipped for this posting
- skipper.init(skipFP,
- docIndex,
- freqIndex,
- posIndex,
- 0,
- docFreq,
- storePayloads);
- skipper.setIndexOptions(indexOptions);
-
- skipped = true;
+ public override void DecodeTerm(long[] empty, DataInput input, FieldInfo fieldInfo, BlockTermState bTermState,
+ bool absolute)
+ {
+ var termState = (SepTermState) bTermState;
+ termState.DOC_INDEX.Read(input, absolute);
+ if (fieldInfo.FieldIndexOptions != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ termState.FREQ_INDEX.Read(input, absolute);
+ if (fieldInfo.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ termState.POS_INDEX.Read(input, absolute);
+
+ if (fieldInfo.HasPayloads())
+ {
+ if (absolute)
+ {
+ termState.PAYLOAD_FP = input.ReadVLong();
+ }
+ else
+ {
+ termState.PAYLOAD_FP += input.ReadVLong();
+ }
+ }
+ }
+ }
+
+ if (termState.DocFreq >= _skipMinimum)
+ {
+ if (absolute)
+ {
+ termState.SKIP_FP = input.ReadVLong();
+ }
+ else
+ {
+ termState.SKIP_FP += input.ReadVLong();
+ }
+ }
+ else if (absolute)
+ {
+ termState.SKIP_FP = 0;
+ }
}
- final int newCount = skipper.skipTo(target);
-
- if (newCount > count) {
-
- // Skipper did move
- if (!omitTF) {
- skipper.getFreqIndex().seek(freqReader);
- }
- skipper.getDocIndex().seek(docReader);
- count = newCount;
- doc = accum = skipper.getDoc();
- }
- }
-
- // Now, linear scan for the rest:
- do {
- if (nextDoc() == NO_MORE_DOCS) {
- return NO_MORE_DOCS;
+ public override DocsEnum Docs(FieldInfo fieldInfo, BlockTermState bTermState, Bits liveDocs, DocsEnum reuse,
+ int flags)
+ {
+ var termState = (SepTermState)bTermState;
+
+ SepDocsEnum docsEnum;
+ if (!(reuse is SepDocsEnum))
+ {
+ docsEnum = new SepDocsEnum(this);
+ }
+ else
+ {
+ docsEnum = (SepDocsEnum) reuse;
+ if (docsEnum.START_DOC_IN != _docIn)
+ {
+ // If you are using ParellelReader, and pass in a
+ // reused DocsAndPositionsEnum, it could have come
+ // from another reader also using sep codec
+ docsEnum = new SepDocsEnum(this);
+ }
+ }
+
+ return docsEnum.Init(fieldInfo, termState, liveDocs);
}
- } while (target > doc);
-
- return doc;
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- class SepDocsAndPositionsEnum extends DocsAndPositionsEnum {
- int docFreq;
- int doc = -1;
- int accum;
- int count;
- int freq;
- long freqStart;
-
- private bool storePayloads;
- private Bits liveDocs;
- private final IntIndexInput.Reader docReader;
- private final IntIndexInput.Reader freqReader;
- private final IntIndexInput.Reader posReader;
- private final IndexInput payloadIn;
- private long skipFP;
-
- private final IntIndexInput.Index docIndex;
- private final IntIndexInput.Index freqIndex;
- private final IntIndexInput.Index posIndex;
- private final IntIndexInput startDocIn;
-
- private long payloadFP;
-
- private int pendingPosCount;
- private int position;
- private int payloadLength;
- private long pendingPayloadBytes;
-
- private bool skipped;
- private SepSkipListReader skipper;
- private bool payloadPending;
- private bool posSeekPending;
-
- SepDocsAndPositionsEnum() {
- startDocIn = docIn;
- docReader = docIn.reader();
- docIndex = docIn.index();
- freqReader = freqIn.reader();
- freqIndex = freqIn.index();
- posReader = posIn.reader();
- posIndex = posIn.index();
- payloadIn = SepPostingsReader.this.payloadIn.clone();
- }
-
- SepDocsAndPositionsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) {
- this.liveDocs = liveDocs;
- storePayloads = fieldInfo.hasPayloads();
- //System.out.println("Sep D&P init");
-
- // TODO: can't we only do this if consumer
- // skipped consuming the previous docs?
- docIndex.copyFrom(termState.docIndex);
- docIndex.seek(docReader);
- //System.out.println(" docIndex=" + docIndex);
-
- freqIndex.copyFrom(termState.freqIndex);
- freqIndex.seek(freqReader);
- //System.out.println(" freqIndex=" + freqIndex);
-
- posIndex.copyFrom(termState.posIndex);
- //System.out.println(" posIndex=" + posIndex);
- posSeekPending = true;
- payloadPending = false;
-
- payloadFP = termState.payloadFP;
- skipFP = termState.skipFP;
- //System.out.println(" skipFP=" + skipFP);
-
- docFreq = termState.docFreq;
- count = 0;
- doc = -1;
- accum = 0;
- pendingPosCount = 0;
- pendingPayloadBytes = 0;
- skipped = false;
-
- return this;
- }
-
- @Override
- public int nextDoc() {
- while(true) {
- if (count == docFreq) {
- return doc = NO_MORE_DOCS;
+ public override DocsAndPositionsEnum DocsAndPositions(FieldInfo fieldInfo, BlockTermState bTermState,
+ Bits liveDocs, DocsAndPositionsEnum reuse, int flags)
+ {
+
+ Debug.Assert(fieldInfo.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ var termState = (SepTermState)bTermState;
+ SepDocsAndPositionsEnum postingsEnum;
+ if (!(reuse is SepDocsAndPositionsEnum))
+ {
+ postingsEnum = new SepDocsAndPositionsEnum(this);
+ }
+ else
+ {
+ postingsEnum = (SepDocsAndPositionsEnum) reuse;
+ if (postingsEnum.START_DOC_IN != _docIn)
+ {
+ // If you are using ParellelReader, and pass in a
+ // reused DocsAndPositionsEnum, it could have come
+ // from another reader also using sep codec
+ postingsEnum = new SepDocsAndPositionsEnum(this);
+ }
+ }
+
+ return postingsEnum.Init(fieldInfo, termState, liveDocs);
}
- count++;
-
- // TODO: maybe we should do the 1-bit trick for encoding
- // freq=1 case?
-
- // Decode next doc
- //System.out.println(" sep d&p read doc");
- accum += docReader.next();
-
- //System.out.println(" sep d&p read freq");
- freq = freqReader.next();
-
- pendingPosCount += freq;
-
- if (liveDocs == null || liveDocs.get(accum)) {
- break;
+ public override long RamBytesUsed()
+ {
+ return 0;
}
- }
-
- position = 0;
- return (doc = accum);
- }
-
- @Override
- public int freq() {
- return freq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public int advance(int target) {
- //System.out.println("SepD&P advance target=" + target + " vs current=" + doc + " this=" + this);
-
- if ((target - skipInterval) >= doc && docFreq >= skipMinimum) {
- // There are enough docs in the posting to have
- // skip data, and its not too close
-
- if (skipper == null) {
- //System.out.println(" create skipper");
- // This DocsEnum has never done any skipping
- skipper = new SepSkipListReader(skipIn.clone(),
- freqIn,
- docIn,
- posIn,
- maxSkipLevels, skipInterval);
+ public override void CheckIntegrity()
+ {
+ // TODO: remove sep layout, its fallen behind on features...
}
- if (!skipped) {
- //System.out.println(" init skip data skipFP=" + skipFP);
- // We haven't yet skipped for this posting
- skipper.init(skipFP,
- docIndex,
- freqIndex,
- posIndex,
- payloadFP,
- docFreq,
- storePayloads);
- skipper.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- skipped = true;
- }
- final int newCount = skipper.skipTo(target);
- //System.out.println(" skip newCount=" + newCount + " vs " + count);
-
- if (newCount > count) {
-
- // Skipper did move
- skipper.getFreqIndex().seek(freqReader);
- skipper.getDocIndex().seek(docReader);
- //System.out.println(" doc seek'd to " + skipper.getDocIndex());
- // NOTE: don't seek pos here; do it lazily
- // instead. Eg a PhraseQuery may skip to many
- // docs before finally asking for positions...
- posIndex.copyFrom(skipper.getPosIndex());
- posSeekPending = true;
- count = newCount;
- doc = accum = skipper.getDoc();
- //System.out.println(" moved to doc=" + doc);
- //payloadIn.seek(skipper.getPayloadPointer());
- payloadFP = skipper.getPayloadPointer();
- pendingPosCount = 0;
- pendingPayloadBytes = 0;
- payloadPending = false;
- payloadLength = skipper.getPayloadLength();
- //System.out.println(" move payloadLen=" + payloadLength);
+ internal sealed class SepTermState : BlockTermState
+ {
+ // We store only the seek point to the docs file because
+ // the rest of the info (freqIndex, posIndex, etc.) is
+ // stored in the docs file:
+ internal IntIndexInputIndex DOC_INDEX;
+ internal IntIndexInputIndex POS_INDEX;
+ internal IntIndexInputIndex FREQ_INDEX;
+ internal long PAYLOAD_FP;
+ internal long SKIP_FP;
+
+ public override object Clone()
+ {
+ var other = new SepTermState();
+ other.CopyFrom(this);
+ return other;
+ }
+
+ public override void CopyFrom(TermState tsOther)
+ {
+ base.CopyFrom(tsOther);
+
+ var other = (SepTermState)tsOther;
+ if (DOC_INDEX == null)
+ {
+ DOC_INDEX = other.DOC_INDEX.Clone();
+ }
+ else
+ {
+ DOC_INDEX.CopyFrom(other.DOC_INDEX);
+ }
+ if (other.FREQ_INDEX != null)
+ {
+ if (FREQ_INDEX == null)
+ {
+ FREQ_INDEX = other.FREQ_INDEX.Clone();
+ }
+ else
+ {
+ FREQ_INDEX.CopyFrom(other.FREQ_INDEX);
+ }
+ }
+ else
+ {
+ FREQ_INDEX = null;
+ }
+ if (other.POS_INDEX != null)
+ {
+ if (POS_INDEX == null)
+ {
+ POS_INDEX = other.POS_INDEX.Clone();
+ }
+ else
+ {
+ POS_INDEX.CopyFrom(other.POS_INDEX);
+ }
+ }
+ else
+ {
+ POS_INDEX = null;
+ }
+ PAYLOAD_FP = other.PAYLOAD_FP;
+ SKIP_FP = other.SKIP_FP;
+ }
+
+ public override string ToString()
+ {
+ return base.ToString() + " docIndex=" + DOC_INDEX + " freqIndex=" + FREQ_INDEX + " posIndex=" + POS_INDEX +
+ " payloadFP=" + PAYLOAD_FP + " skipFP=" + SKIP_FP;
+ }
}
- }
-
- // Now, linear scan for the rest:
- do {
- if (nextDoc() == NO_MORE_DOCS) {
- //System.out.println(" advance nextDoc=END");
- return NO_MORE_DOCS;
- }
- //System.out.println(" advance nextDoc=" + doc);
- } while (target > doc);
- //System.out.println(" return doc=" + doc);
- return doc;
- }
- @Override
- public int nextPosition() {
- if (posSeekPending) {
- posIndex.seek(posReader);
- payloadIn.seek(payloadFP);
- posSeekPending = false;
- }
-
- // scan over any docs that were iterated without their
- // positions
- while (pendingPosCount > freq) {
- final int code = posReader.next();
- if (storePayloads && (code & 1) != 0) {
- // Payload length has changed
- payloadLength = posReader.next();
- Debug.Assert( payloadLength >= 0;
+ internal class SepDocsEnum : DocsEnum
+ {
+ private readonly SepPostingsReader _outerInstance;
+
+ private int _docFreq;
+ private int _doc = -1;
+ private int _accum;
+ private int _count;
+ private int _freq;
+
+ // TODO: -- should we do omitTF with 2 different enum classes?
+ private bool _omitTf;
+ private FieldInfo.IndexOptions _indexOptions;
+ private bool _storePayloads;
+ private Bits _liveDocs;
+ private readonly IntIndexInputReader _docReader;
+ private readonly IntIndexInputReader _freqReader;
+ private long _skipFp;
+
+ private readonly IntIndexInputIndex _docIndex;
+ private readonly IntIndexInputIndex _freqIndex;
+ private readonly IntIndexInputIndex _posIndex;
+
+ // TODO: -- should we do hasProx with 2 different enum classes?
+
+ private bool _skipped;
+ private SepSkipListReader _skipper;
+
+ internal IntIndexInput START_DOC_IN;
+
+ internal SepDocsEnum(SepPostingsReader outerInstance)
+ {
+ _outerInstance = outerInstance;
+ _docReader = outerInstance._docIn.Reader();
+ _docIndex = outerInstance._docIn.Index();
+ if (outerInstance._freqIn != null)
+ {
+ _freqReader = outerInstance._freqIn.Reader();
+ _freqIndex = outerInstance._freqIn.Index();
+ }
+ else
+ {
+ _freqReader = null;
+ _freqIndex = null;
+ }
+ _posIndex = outerInstance._posIn != null ? outerInstance._posIn.Index() : null;
+
+ START_DOC_IN = outerInstance._docIn;
+ }
+
+ internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
+ {
+ _liveDocs = liveDocs;
+ if (fieldInfo.FieldIndexOptions.HasValue)
+ _indexOptions = fieldInfo.FieldIndexOptions.Value;
+
+ _omitTf = _indexOptions == FieldInfo.IndexOptions.DOCS_ONLY;
+ _storePayloads = fieldInfo.HasPayloads();
+
+ // TODO: can't we only do this if consumer
+ // skipped consuming the previous docs?
+ _docIndex.CopyFrom(termState.DOC_INDEX);
+ _docIndex.Seek(_docReader);
+
+ if (!_omitTf)
+ {
+ _freqIndex.CopyFrom(termState.FREQ_INDEX);
+ _freqIndex.Seek(_freqReader);
+ }
+
+ _docFreq = termState.DocFreq;
+ // NOTE: unused if docFreq < skipMinimum:
+ _skipFp = termState.SKIP_FP;
+ _count = 0;
+ _doc = -1;
+ _accum = 0;
+ _freq = 1;
+ _skipped = false;
+
+ return this;
+ }
+
+ public override int NextDoc()
+ {
+
+ while (true)
+ {
+ if (_count == _docFreq)
+ {
+ return _doc = NO_MORE_DOCS;
+ }
+
+ _count++;
+
+ // Decode next doc
+ //System.out.println("decode docDelta:");
+ _accum += _docReader.Next();
+
+ if (!_omitTf)
+ {
+ //System.out.println("decode freq:");
+ _freq = _freqReader.Next();
+ }
+
+ if (_liveDocs == null || _liveDocs.Get(_accum))
+ {
+ break;
+ }
+ }
+ return (_doc = _accum);
+ }
+
+ public override int Freq()
+ {
+ return _freq;
+ }
+
+ public override int DocID()
+ {
+ return _doc;
+ }
+
+ public override int Advance(int target)
+ {
+
+ if ((target - _outerInstance._skipInterval) >= _doc && _docFreq >= _outerInstance._skipMinimum)
+ {
+
+ // There are enough docs in the posting to have
+ // skip data, and its not too close
+
+ if (_skipper == null)
+ {
+ // This DocsEnum has never done any skipping
+ _skipper = new SepSkipListReader((IndexInput) _outerInstance._skipIn.Clone(),
+ _outerInstance._freqIn,
+ _outerInstance._docIn, _outerInstance._posIn, _outerInstance._maxSkipLevels,
+ _outerInstance._skipInterval);
+
+ }
+
+ if (!_skipped)
+ {
+ // We haven't yet skipped for this posting
+ _skipper.Init(_skipFp, _docIndex, _freqIndex, _posIndex, 0, _docFreq, _storePayloads);
+ _skipper.IndexOptions = _indexOptions;
+
+ _skipped = true;
+ }
+
+ int newCount = _skipper.SkipTo(target);
+
+ if (newCount > _count)
+ {
+
+ // Skipper did move
+ if (!_omitTf)
+ {
+ _skipper.FreqIndex.Seek(_freqReader);
+ }
+ _skipper.DocIndex.Seek(_docReader);
+ _count = newCount;
+ _doc = _accum = _skipper.Doc;
+ }
+ }
+
+ // Now, linear scan for the rest:
+ do
+ {
+ if (NextDoc() == NO_MORE_DOCS)
+ {
+ return NO_MORE_DOCS;
+ }
+ } while (target > _doc);
+
+ return _doc;
+ }
+
+ public override long Cost()
+ {
+ return _docFreq;
+ }
}
- pendingPosCount--;
- position = 0;
- pendingPayloadBytes += payloadLength;
- }
-
- final int code = posReader.next();
-
- if (storePayloads) {
- if ((code & 1) != 0) {
- // Payload length has changed
- payloadLength = posReader.next();
- Debug.Assert( payloadLength >= 0;
- }
- position += code >>> 1;
- pendingPayloadBytes += payloadLength;
- payloadPending = payloadLength > 0;
- } else {
- position += code;
- }
-
- pendingPosCount--;
- Debug.Assert( pendingPosCount >= 0;
- return position;
- }
-
- @Override
- public int startOffset() {
- return -1;
- }
- @Override
- public int endOffset() {
- return -1;
- }
-
- private BytesRef payload;
-
- @Override
- public BytesRef getPayload() {
- if (!payloadPending) {
- return null;
- }
-
- if (pendingPayloadBytes == 0) {
- return payload;
- }
-
- Debug.Assert( pendingPayloadBytes >= payloadLength;
-
- if (pendingPayloadBytes > payloadLength) {
- payloadIn.seek(payloadIn.getFilePointer() + (pendingPayloadBytes - payloadLength));
- }
-
- if (payload == null) {
- payload = new BytesRef();
- payload.bytes = new byte[payloadLength];
- } else if (payload.bytes.length < payloadLength) {
- payload.grow(payloadLength);
- }
-
- payloadIn.readBytes(payload.bytes, 0, payloadLength);
- payload.length = payloadLength;
- pendingPayloadBytes = 0;
- return payload;
- }
-
- @Override
- public long cost() {
- return docFreq;
+ internal class SepDocsAndPositionsEnum : DocsAndPositionsEnum
+ {
+ private readonly SepPostingsReader _outerInstance;
+ private BytesRef _payload;
+
+ private int _docFreq;
+ private int _doc = -1;
+ private int _accum;
+ private int _count;
+ private int _freq;
+
+ private bool _storePayloads;
+ private Bits _liveDocs;
+ private readonly IntIndexInputReader _docReader;
+ private readonly IntIndexInputReader _freqReader;
+ private readonly IntIndexInputReader _posReader;
+ private readonly IndexInput _payloadIn;
+ private long _skipFp;
+
+ private readonly IntIndexInputIndex _docIndex;
+ private readonly IntIndexInputIndex _freqIndex;
+ private readonly IntIndexInputIndex _posIndex;
+
+ private long _payloadFp;
+
+ private int _pendingPosCount;
+ private int _position;
+ private int _payloadLength;
+ private long _pendingPayloadBytes;
+
+ private bool _skipped;
+ private SepSkipListReader _skipper;
+ private bool _payloadPending;
+ private bool _posSeekPending;
+
+ internal IntIndexInput START_DOC_IN;
+
+ internal SepDocsAndPositionsEnum(SepPostingsReader outerInstance)
+ {
+ _outerInstance = outerInstance;
+ _docReader = outerInstance._docIn.Reader();
+ _docIndex = outerInstance._docIn.Index();
+ _freqReader = outerInstance._freqIn.Reader();
+ _freqIndex = outerInstance._freqIn.Index();
+ _posReader = outerInstance._posIn.Reader();
+ _posIndex = outerInstance._posIn.Index();
+ _payloadIn = (IndexInput) outerInstance._payloadIn.Clone();
+
+ START_DOC_IN = outerInstance._docIn;
+ }
+
+ internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
+ {
+ _liveDocs = liveDocs;
+ _storePayloads = fieldInfo.HasPayloads();
+
+ // TODO: can't we only do this if consumer skipped consuming the previous docs?
+ _docIndex.CopyFrom(termState.DOC_INDEX);
+ _docIndex.Seek(_docReader);
+
+ _freqIndex.CopyFrom(termState.FREQ_INDEX);
+ _freqIndex.Seek(_freqReader);
+
+ _posIndex.CopyFrom(termState.POS_INDEX);
+ _posSeekPending = true;
+ _payloadPending = false;
+
+ _payloadFp = termState.PAYLOAD_FP;
+ _skipFp = termState.SKIP_FP;
+
+ _docFreq = termState.DocFreq;
+ _count = 0;
+ _doc = -1;
+ _accum = 0;
+ _pendingPosCount = 0;
+ _pendingPayloadBytes = 0;
+ _skipped = false;
+
+ return this;
+ }
+
+ public override int NextDoc()
+ {
+ while (true)
+ {
+ if (_count == _docFreq)
+ return _doc = NO_MORE_DOCS;
+
+ _count++;
+
+ // Decode next doc
+ _accum += _docReader.Next();
+ _freq = _freqReader.Next();
+ _pendingPosCount += _freq;
+
+ if (_liveDocs == null || _liveDocs.Get(_accum))
+ break;
+ }
+
+ _position = 0;
+ return (_doc = _accum);
+ }
+
+ public override int Freq()
+ {
+ return _freq;
+ }
+
+ public override int DocID()
+ {
+ return _doc;
+ }
+
+ public override int Advance(int target)
+ {
+ //System.out.println("SepD&P advance target=" + target + " vs current=" + doc + " this=" + this);
+
+ if ((target - _outerInstance._skipInterval) >= _doc && _docFreq >= _outerInstance._skipMinimum)
+ {
+
+ // There are enough docs in the posting to have
+ // skip data, and its not too close
+
+ if (_skipper == null)
+ {
+ //System.out.println(" create skipper");
+ // This DocsEnum has never done any skipping
+ _skipper = new SepSkipListReader((IndexInput) _outerInstance._skipIn.Clone(),
+ _outerInstance._freqIn,
+ _outerInstance._docIn, _outerInstance._posIn, _outerInstance._maxSkipLevels,
+ _outerInstance._skipInterval);
+ }
+
+ if (!_skipped)
+ {
+ //System.out.println(" init skip data skipFP=" + skipFP);
+ // We haven't yet skipped for this posting
+ _skipper.Init(_skipFp, _docIndex, _freqIndex, _posIndex, _payloadFp, _docFreq, _storePayloads);
+ _skipper.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ _skipped = true;
+ }
+
+ int newCount = _skipper.SkipTo(target);
+
+ if (newCount > _count)
+ {
+
+ // Skipper did move
+ _skipper.FreqIndex.Seek(_freqReader);
+ _skipper.DocIndex.Seek(_docReader);
+
+ // NOTE: don't seek pos here; do it lazily
+ // instead. Eg a PhraseQuery may skip to many
+ // docs before finally asking for positions...
+
+ _posIndex.CopyFrom(_skipper.PosIndex);
+ _posSeekPending = true;
+ _count = newCount;
+ _doc = _accum = _skipper.Doc;
+
+ _payloadFp = _skipper.PayloadPointer;
+ _pendingPosCount = 0;
+ _pendingPayloadBytes = 0;
+ _payloadPending = false;
+ _payloadLength = _skipper.PayloadLength;
+ }
+ }
+
+ // Now, linear scan for the rest:
+ do
+ {
+ if (NextDoc() == NO_MORE_DOCS)
+ {
+ return NO_MORE_DOCS;
+ }
+
+ } while (target > _doc);
+
+ return _doc;
+ }
+
+ public override int NextPosition()
+ {
+ if (_posSeekPending)
+ {
+ _posIndex.Seek(_posReader);
+ _payloadIn.Seek(_payloadFp);
+ _posSeekPending = false;
+ }
+
+ int code;
+
+ // scan over any docs that were iterated without their positions
+ while (_pendingPosCount > _freq)
+ {
+ code = _posReader.Next();
+ if (_storePayloads && (code & 1) != 0)
+ {
+ // Payload length has changed
+ _payloadLength = _posReader.Next();
+ Debug.Assert(_payloadLength >= 0);
+ }
+ _pendingPosCount--;
+ _position = 0;
+ _pendingPayloadBytes += _payloadLength;
+ }
+
+ code = _posReader.Next();
+
+ if (_storePayloads)
+ {
+ if ((code & 1) != 0)
+ {
+ // Payload length has changed
+ _payloadLength = _posReader.Next();
+ Debug.Assert(_payloadLength >= 0);
+ }
+ _position += (int) ((uint) code >> 1);
+ _pendingPayloadBytes += _payloadLength;
+ _payloadPending = _payloadLength > 0;
+ }
+ else
+ {
+ _position += code;
+ }
+
+ _pendingPosCount--;
+ Debug.Assert(_pendingPosCount >= 0);
+ return _position;
+ }
+
+ public override int StartOffset()
+ {
+ return -1;
+ }
+
+ public override int EndOffset()
+ {
+ return -1;
+ }
+
+ public override BytesRef Payload
+ {
+ get
+ {
+ if (!_payloadPending)
+ {
+ return null;
+ }
+
+ if (_pendingPayloadBytes == 0)
+ {
+ return _payload;
+ }
+
+ Debug.Assert(_pendingPayloadBytes >= _payloadLength);
+
+ if (_pendingPayloadBytes > _payloadLength)
+ {
+ _payloadIn.Seek(_payloadIn.FilePointer + (_pendingPayloadBytes - _payloadLength));
+ }
+
+ if (_payload == null)
+ {
+ _payload = new BytesRef {Bytes = new sbyte[_payloadLength]};
+ }
+ else if (_payload.Bytes.Length < _payloadLength)
+ {
+ _payload.Grow(_payloadLength);
+ }
+
+ _payloadIn.ReadBytes(_payload.Bytes, 0, _payloadLength);
+ _payload.Length = _payloadLength;
+ _pendingPayloadBytes = 0;
+ return _payload;
+ }
+ }
+
+ public override long Cost()
+ {
+ return _docFreq;
+ }
+ }
}
- }
-
- @Override
- public long ramBytesUsed() {
- return 0;
- }
-
- @Override
- public void checkIntegrity() {
- // TODO: remove sep layout, its fallen behind on features...
- }
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/SepPostingsWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/SepPostingsWriter.cs b/src/Lucene.Net.Codecs/Sep/SepPostingsWriter.cs
index eba7022..602f724 100644
--- a/src/Lucene.Net.Codecs/Sep/SepPostingsWriter.cs
+++ b/src/Lucene.Net.Codecs/Sep/SepPostingsWriter.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,355 +15,397 @@ package codecs.sep;
* limitations under the License.
*/
-import java.io.IOException;
-
-import codecs.BlockTermState;
-import codecs.CodecUtil;
-import codecs.PostingsWriterBase;
-import index.CorruptIndexException;
-import index.DocsEnum;
-import index.FieldInfo;
-import index.FieldInfo.IndexOptions;
-import index.IndexFileNames;
-import index.SegmentWriteState;
-import store.DataOutput;
-import store.IndexOutput;
-import store.RAMOutputStream;
-import util.BytesRef;
-import util.IOUtils;
-
-/** Writes frq to .frq, docs to .doc, pos to .pos, payloads
- * to .pyl, skip data to .skp
- *
- * @lucene.experimental */
-public final class SepPostingsWriter extends PostingsWriterBase {
- final static String CODEC = "SepPostingsWriter";
-
- final static String DOC_EXTENSION = "doc";
- final static String SKIP_EXTENSION = "skp";
- final static String FREQ_EXTENSION = "frq";
- final static String POS_EXTENSION = "pos";
- final static String PAYLOAD_EXTENSION = "pyl";
-
- // Increment version to change it:
- final static int VERSION_START = 0;
- final static int VERSION_CURRENT = VERSION_START;
-
- IntIndexOutput freqOut;
- IntIndexOutput.Index freqIndex;
-
- IntIndexOutput posOut;
- IntIndexOutput.Index posIndex;
-
- IntIndexOutput docOut;
- IntIndexOutput.Index docIndex;
-
- IndexOutput payloadOut;
-
- IndexOutput skipOut;
-
- final SepSkipListWriter skipListWriter;
- /** Expert: The fraction of TermDocs entries stored in skip tables,
- * used to accelerate {@link DocsEnum#advance(int)}. Larger values result in
- * smaller indexes, greater acceleration, but fewer accelerable cases, while
- * smaller values result in bigger indexes, less acceleration and more
- * accelerable cases. More detailed experiments would be useful here. */
- final int skipInterval;
- static final int DEFAULT_SKIP_INTERVAL = 16;
-
- /**
- * Expert: minimum docFreq to write any skip data at all
- */
- final int skipMinimum;
-
- /** Expert: The maximum number of skip levels. Smaller values result in
- * slightly smaller indexes, but slower skipping in big posting lists.
- */
- final int maxSkipLevels = 10;
-
- final int totalNumDocs;
-
- bool storePayloads;
- IndexOptions indexOptions;
-
- FieldInfo fieldInfo;
-
- int lastPayloadLength;
- int lastPosition;
- long payloadStart;
- int lastDocID;
- int df;
-
- SepTermState lastState;
- long lastPayloadFP;
- long lastSkipFP;
-
- public SepPostingsWriter(SegmentWriteState state, IntStreamFactory factory) {
- this(state, factory, DEFAULT_SKIP_INTERVAL);
- }
-
- public SepPostingsWriter(SegmentWriteState state, IntStreamFactory factory, int skipInterval) {
- freqOut = null;
- freqIndex = null;
- posOut = null;
- posIndex = null;
- payloadOut = null;
- bool success = false;
- try {
- this.skipInterval = skipInterval;
- this.skipMinimum = skipInterval; /* set to the same for now */
- final String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, DOC_EXTENSION);
-
- docOut = factory.createOutput(state.directory, docFileName, state.context);
- docIndex = docOut.index();
-
- if (state.fieldInfos.hasFreq()) {
- final String frqFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FREQ_EXTENSION);
- freqOut = factory.createOutput(state.directory, frqFileName, state.context);
- freqIndex = freqOut.index();
- }
-
- if (state.fieldInfos.hasProx()) {
- final String posFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, POS_EXTENSION);
- posOut = factory.createOutput(state.directory, posFileName, state.context);
- posIndex = posOut.index();
-
- // TODO: -- only if at least one field stores payloads?
- final String payloadFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, PAYLOAD_EXTENSION);
- payloadOut = state.directory.createOutput(payloadFileName, state.context);
- }
-
- final String skipFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SKIP_EXTENSION);
- skipOut = state.directory.createOutput(skipFileName, state.context);
-
- totalNumDocs = state.segmentInfo.getDocCount();
-
- skipListWriter = new SepSkipListWriter(skipInterval,
- maxSkipLevels,
- totalNumDocs,
- freqOut, docOut,
- posOut, payloadOut);
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docOut, skipOut, freqOut, posOut, payloadOut);
- }
- }
- }
-
- @Override
- public void init(IndexOutput termsOut) {
- CodecUtil.writeHeader(termsOut, CODEC, VERSION_CURRENT);
- // TODO: -- just ask skipper to "start" here
- termsOut.writeInt(skipInterval); // write skipInterval
- termsOut.writeInt(maxSkipLevels); // write maxSkipLevels
- termsOut.writeInt(skipMinimum); // write skipMinimum
- }
-
- @Override
- public BlockTermState newTermState() {
- return new SepTermState();
- }
-
- @Override
- public void startTerm() {
- docIndex.mark();
- //System.out.println("SEPW: startTerm docIndex=" + docIndex);
-
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- freqIndex.mark();
- }
-
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- posIndex.mark();
- payloadStart = payloadOut.getFilePointer();
- lastPayloadLength = -1;
- }
+namespace Lucene.Net.Codecs.Sep
+{
+ using System.Diagnostics;
+ using Index;
+ using Store;
+ using Util;
+
+ /// <summary>
+ /// Writes frq to .frq, docs to .doc, pos to .pos, payloads
+ /// to .pyl, skip data to .skp
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public sealed class SepPostingsWriter : PostingsWriterBase
+ {
+ internal const string CODEC = "SepPostingsWriter";
+
+ internal const string DOC_EXTENSION = "doc";
+ internal const string SKIP_EXTENSION = "skp";
+ internal const string FREQ_EXTENSION = "frq";
+ internal const string POS_EXTENSION = "pos";
+ internal const string PAYLOAD_EXTENSION = "pyl";
+
+ // Increment version to change it:
+ internal const int VERSION_START = 0;
+ internal const int VERSION_CURRENT = VERSION_START;
+
+ internal IntIndexOutput FREQ_OUT;
+ internal IntIndexOutputIndex FREQ_INDEX;
+
+ internal IntIndexOutput POS_OUT;
+ internal IntIndexOutputIndex POS_INDEX;
+
+ internal IntIndexOutput DOC_OUT;
+ internal IntIndexOutputIndex DOC_INDEX;
+
+ internal IndexOutput PAYLOAD_OUT;
+
+ internal IndexOutput SKIP_OUT;
+
+ internal readonly SepSkipListWriter SKIP_LIST_WRITER;
+
+ /// <summary>
+ /// Expert: The fraction of TermDocs entries stored in skip tables,
+ /// used to accelerate <seealso cref="DocsEnum#advance(int)"/>. Larger values result in
+ /// smaller indexes, greater acceleration, but fewer accelerable cases, while
+ /// smaller values result in bigger indexes, less acceleration and more
+ /// accelerable cases. More detailed experiments would be useful here.
+ /// </summary>
+ internal readonly int SKIP_INTERVAL;
+
+ internal const int DEFAULT_SKIP_INTERVAL = 16;
+
+ /// <summary>
+ /// Expert: minimum docFreq to write any skip data at all
+ /// </summary>
+ internal readonly int SKIP_MINIMUM;
+
+ /// <summary>
+ /// Expert: The maximum number of skip levels. Smaller values result in
+ /// slightly smaller indexes, but slower skipping in big posting lists.
+ /// </summary>
+ internal readonly int MAX_SKIP_LEVELS = 10;
+
+ internal readonly int TOTAL_NUM_DOCS;
+
+ internal bool STORE_PAYLOADS;
+ internal FieldInfo.IndexOptions INDEX_OPTIONS;
+
+ internal FieldInfo FIELD_INFO;
+
+ internal int LAST_PAYLOAD_LENGTH;
+ internal int LAST_POSITION;
+ internal long PAYLOAD_START;
+ internal int LAST_DOC_ID;
+ internal int DF;
+
+ private SepTermState _lastState;
+ internal long LAST_PAYLOAD_FP;
+ internal long LAST_SKIP_FP;
+
+ public SepPostingsWriter(SegmentWriteState state, IntStreamFactory factory)
+ : this(state, factory, DEFAULT_SKIP_INTERVAL)
+ {
+ }
- skipListWriter.resetSkip(docIndex, freqIndex, posIndex);
- }
-
- // Currently, this instance is re-used across fields, so
- // our parent calls setField whenever the field changes
- @Override
- public int setField(FieldInfo fieldInfo) {
- this.fieldInfo = fieldInfo;
- this.indexOptions = fieldInfo.getIndexOptions();
- if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
- throw new UnsupportedOperationException("this codec cannot index offsets");
- }
- skipListWriter.setIndexOptions(indexOptions);
- storePayloads = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && fieldInfo.hasPayloads();
- lastPayloadFP = 0;
- lastSkipFP = 0;
- lastState = setEmptyState();
- return 0;
- }
-
- private SepTermState setEmptyState() {
- SepTermState emptyState = new SepTermState();
- emptyState.docIndex = docOut.index();
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- emptyState.freqIndex = freqOut.index();
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- emptyState.posIndex = posOut.index();
- }
- }
- emptyState.payloadFP = 0;
- emptyState.skipFP = 0;
- return emptyState;
- }
+ public SepPostingsWriter(SegmentWriteState state, IntStreamFactory factory, int skipInterval)
+ {
+ FREQ_OUT = null;
+ FREQ_INDEX = null;
+ POS_OUT = null;
+ POS_INDEX = null;
+ PAYLOAD_OUT = null;
+ var success = false;
+ try
+ {
+ SKIP_INTERVAL = skipInterval;
+ SKIP_MINIMUM = skipInterval; // set to the same for now
+ var docFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, DOC_EXTENSION);
+
+ DOC_OUT = factory.CreateOutput(state.Directory, docFileName, state.Context);
+ DOC_INDEX = DOC_OUT.Index();
+
+ if (state.FieldInfos.HasFreq())
+ {
+ var frqFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FREQ_EXTENSION);
+ FREQ_OUT = factory.CreateOutput(state.Directory, frqFileName, state.Context);
+ FREQ_INDEX = FREQ_OUT.Index();
+ }
+
+ if (state.FieldInfos.HasProx())
+ {
+ var posFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, POS_EXTENSION);
+ POS_OUT = factory.CreateOutput(state.Directory, posFileName, state.Context);
+ POS_INDEX = POS_OUT.Index();
+
+ // TODO: -- only if at least one field stores payloads?
+ var payloadFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,PAYLOAD_EXTENSION);
+ PAYLOAD_OUT = state.Directory.CreateOutput(payloadFileName, state.Context);
+ }
+
+ var skipFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, SKIP_EXTENSION);
+ SKIP_OUT = state.Directory.CreateOutput(skipFileName, state.Context);
+
+ TOTAL_NUM_DOCS = state.SegmentInfo.DocCount;
+
+ SKIP_LIST_WRITER = new SepSkipListWriter(skipInterval, MAX_SKIP_LEVELS, TOTAL_NUM_DOCS, FREQ_OUT, DOC_OUT,
+ POS_OUT, PAYLOAD_OUT);
+
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.CloseWhileHandlingException(DOC_OUT, SKIP_OUT, FREQ_OUT, POS_OUT, PAYLOAD_OUT);
+ }
+ }
+ }
+ public override void Init(IndexOutput termsOut)
+ {
+ CodecUtil.WriteHeader(termsOut, CODEC, VERSION_CURRENT);
+ // TODO: -- just ask skipper to "start" here
+ termsOut.WriteInt(SKIP_INTERVAL); // write skipInterval
+ termsOut.WriteInt(MAX_SKIP_LEVELS); // write maxSkipLevels
+ termsOut.WriteInt(SKIP_MINIMUM); // write skipMinimum
+ }
- /** Adds a new doc in this term. If this returns null
- * then we just skip consuming positions/payloads. */
- @Override
- public void startDoc(int docID, int termDocFreq) {
+ public override BlockTermState NewTermState()
+ {
+ return new SepTermState();
+ }
- final int delta = docID - lastDocID;
- //System.out.println("SEPW: startDoc: write doc=" + docID + " delta=" + delta + " out.fp=" + docOut);
+ public override void StartTerm()
+ {
+ DOC_INDEX.Mark();
+
+ if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ FREQ_INDEX.Mark();
+ }
+
+ if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ POS_INDEX.Mark();
+ PAYLOAD_START = PAYLOAD_OUT.FilePointer;
+ LAST_PAYLOAD_LENGTH = -1;
+ }
+
+ SKIP_LIST_WRITER.ResetSkip(DOC_INDEX, FREQ_INDEX, POS_INDEX);
+ }
- if (docID < 0 || (df > 0 && delta <= 0)) {
- throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " ) (docOut: " + docOut + ")");
- }
+ // Currently, this instance is re-used across fields, so
+ // our parent calls setField whenever the field changes
+ public override int SetField(FieldInfo fi)
+ {
+ FIELD_INFO = fi;
+
+ if (FIELD_INFO.FieldIndexOptions.HasValue)
+ INDEX_OPTIONS = FIELD_INFO.FieldIndexOptions.Value;
+
+ if (INDEX_OPTIONS >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ {
+ throw new System.NotSupportedException("this codec cannot index offsets");
+ }
+ SKIP_LIST_WRITER.IndexOptions = INDEX_OPTIONS;
+ STORE_PAYLOADS = INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS &&
+ FIELD_INFO.HasPayloads();
+ LAST_PAYLOAD_FP = 0;
+ LAST_SKIP_FP = 0;
+ _lastState = SetEmptyState();
+ return 0;
+ }
- if ((++df % skipInterval) == 0) {
- // TODO: -- awkward we have to make these two
- // separate calls to skipper
- //System.out.println(" buffer skip lastDocID=" + lastDocID);
- skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength);
- skipListWriter.bufferSkip(df);
- }
+ private SepTermState SetEmptyState()
+ {
+ var emptyState = new SepTermState {DocIndex = DOC_OUT.Index()};
+ if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ emptyState.FreqIndex = FREQ_OUT.Index();
+ if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ emptyState.PosIndex = POS_OUT.Index();
+ }
+ }
+ emptyState.PayloadFp = 0;
+ emptyState.SkipFp = 0;
+ return emptyState;
+ }
- lastDocID = docID;
- docOut.write(delta);
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- //System.out.println(" sepw startDoc: write freq=" + termDocFreq);
- freqOut.write(termDocFreq);
- }
- }
-
- /** Add a new position & payload */
- @Override
- public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) {
- Debug.Assert( indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
-
- final int delta = position - lastPosition;
- Debug.Assert( delta >= 0: "position=" + position + " lastPosition=" + lastPosition; // not quite right (if pos=0 is repeated twice we don't catch it)
- lastPosition = position;
-
- if (storePayloads) {
- final int payloadLength = payload == null ? 0 : payload.length;
- if (payloadLength != lastPayloadLength) {
- lastPayloadLength = payloadLength;
- // TODO: explore whether we get better compression
- // by not storing payloadLength into prox stream?
- posOut.write((delta<<1)|1);
- posOut.write(payloadLength);
- } else {
- posOut.write(delta << 1);
- }
-
- if (payloadLength > 0) {
- payloadOut.writeBytes(payload.bytes, payload.offset, payloadLength);
- }
- } else {
- posOut.write(delta);
- }
+ /// <summary>
+ /// Adds a new doc in this term. If this returns null
+ /// then we just skip consuming positions/payloads.
+ /// </summary>
+ public override void StartDoc(int docId, int termDocFreq)
+ {
+ var delta = docId - LAST_DOC_ID;
+
+ if (docId < 0 || (DF > 0 && delta <= 0))
+ {
+ throw new CorruptIndexException("docs out of order (" + docId + " <= " + LAST_DOC_ID + " ) (docOut: " +
+ DOC_OUT + ")");
+ }
+
+ if ((++DF%SKIP_INTERVAL) == 0)
+ {
+ // TODO: -- awkward we have to make these two separate calls to skipper
+ SKIP_LIST_WRITER.SetSkipData(LAST_DOC_ID, STORE_PAYLOADS, LAST_PAYLOAD_LENGTH);
+ SKIP_LIST_WRITER.BufferSkip(DF);
+ }
+
+ LAST_DOC_ID = docId;
+ DOC_OUT.Write(delta);
+ if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ //System.out.println(" sepw startDoc: write freq=" + termDocFreq);
+ FREQ_OUT.Write(termDocFreq);
+ }
+ }
- lastPosition = position;
- }
-
- /** Called when we are done adding positions & payloads */
- @Override
- public void finishDoc() {
- lastPosition = 0;
- }
-
- private static class SepTermState extends BlockTermState {
- public IntIndexOutput.Index docIndex;
- public IntIndexOutput.Index freqIndex;
- public IntIndexOutput.Index posIndex;
- public long payloadFP;
- public long skipFP;
- }
-
- /** Called when we are done adding docs to this term */
- @Override
- public void finishTerm(BlockTermState _state) {
- SepTermState state = (SepTermState)_state;
- // TODO: -- wasteful we are counting this in two places?
- Debug.Assert( state.docFreq > 0;
- Debug.Assert( state.docFreq == df;
-
- state.docIndex = docOut.index();
- state.docIndex.copyFrom(docIndex, false);
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- state.freqIndex = freqOut.index();
- state.freqIndex.copyFrom(freqIndex, false);
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- state.posIndex = posOut.index();
- state.posIndex.copyFrom(posIndex, false);
- } else {
- state.posIndex = null;
- }
- } else {
- state.freqIndex = null;
- state.posIndex = null;
- }
+ /// <summary>
+ /// Add a new position & payload </summary>
+ public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
+ {
+ Debug.Assert(INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ int delta = position - LAST_POSITION;
+ Debug.Assert(delta >= 0, "position=" + position + " lastPosition=" + LAST_POSITION);
+ // not quite right (if pos=0 is repeated twice we don't catch it)
+ LAST_POSITION = position;
+
+ if (STORE_PAYLOADS)
+ {
+ int payloadLength = payload == null ? 0 : payload.Length;
+ if (payloadLength != LAST_PAYLOAD_LENGTH)
+ {
+ LAST_PAYLOAD_LENGTH = payloadLength;
+ // TODO: explore whether we get better compression
+ // by not storing payloadLength into prox stream?
+ POS_OUT.Write((delta << 1) | 1);
+ POS_OUT.Write(payloadLength);
+ }
+ else
+ {
+ POS_OUT.Write(delta << 1);
+ }
+
+ if (payloadLength > 0 && payload != null)
+ {
+ PAYLOAD_OUT.WriteBytes(payload.Bytes, payload.Offset, payloadLength);
+ }
+ }
+ else
+ {
+ POS_OUT.Write(delta);
+ }
+
+ LAST_POSITION = position;
+ }
- if (df >= skipMinimum) {
- state.skipFP = skipOut.getFilePointer();
- //System.out.println(" skipFP=" + skipFP);
- skipListWriter.writeSkip(skipOut);
- //System.out.println(" numBytes=" + (skipOut.getFilePointer()-skipFP));
- } else {
- state.skipFP = -1;
- }
- state.payloadFP = payloadStart;
-
- lastDocID = 0;
- df = 0;
- }
-
- @Override
- public void encodeTerm(long[] longs, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, bool absolute) {
- SepTermState state = (SepTermState)_state;
- if (absolute) {
- lastSkipFP = 0;
- lastPayloadFP = 0;
- lastState = state;
- }
- lastState.docIndex.copyFrom(state.docIndex, false);
- lastState.docIndex.write(out, absolute);
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- lastState.freqIndex.copyFrom(state.freqIndex, false);
- lastState.freqIndex.write(out, absolute);
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- lastState.posIndex.copyFrom(state.posIndex, false);
- lastState.posIndex.write(out, absolute);
- if (storePayloads) {
- if (absolute) {
- out.writeVLong(state.payloadFP);
- } else {
- out.writeVLong(state.payloadFP - lastPayloadFP);
- }
- lastPayloadFP = state.payloadFP;
+ /// <summary>Called when we are done adding positions & payloads </summary>
+ public override void FinishDoc()
+ {
+ LAST_POSITION = 0;
+ }
+
+ private class SepTermState : BlockTermState
+ {
+ public IntIndexOutputIndex DocIndex { get; set; }
+ public IntIndexOutputIndex FreqIndex { get; set; }
+ public IntIndexOutputIndex PosIndex { get; set; }
+ public long PayloadFp { get; set; }
+ public long SkipFp { get; set; }
+ }
+
+ /// <summary>Called when we are done adding docs to this term </summary>
+ public override void FinishTerm(BlockTermState bstate)
+ {
+ var state = (SepTermState)bstate;
+ // TODO: -- wasteful we are counting this in two places?
+ Debug.Assert(state.DocFreq > 0);
+ Debug.Assert(state.DocFreq == DF);
+
+ state.DocIndex = DOC_OUT.Index();
+ state.DocIndex.CopyFrom(DOC_INDEX, false);
+ if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ state.FreqIndex = FREQ_OUT.Index();
+ state.FreqIndex.CopyFrom(FREQ_INDEX, false);
+ if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ state.PosIndex = POS_OUT.Index();
+ state.PosIndex.CopyFrom(POS_INDEX, false);
+ }
+ else
+ {
+ state.PosIndex = null;
+ }
+ }
+ else
+ {
+ state.FreqIndex = null;
+ state.PosIndex = null;
+ }
+
+ if (DF >= SKIP_MINIMUM)
+ {
+ state.SkipFp = SKIP_OUT.FilePointer;
+ SKIP_LIST_WRITER.WriteSkip(SKIP_OUT);
+ }
+ else
+ {
+ state.SkipFp = -1;
+ }
+ state.PayloadFp = PAYLOAD_START;
+
+ LAST_DOC_ID = 0;
+ DF = 0;
+ }
+
+ public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState bstate, bool absolute)
+ {
+ var state = (SepTermState) bstate;
+ if (absolute)
+ {
+ LAST_SKIP_FP = 0;
+ LAST_PAYLOAD_FP = 0;
+ _lastState = state;
+ }
+ _lastState.DocIndex.CopyFrom(state.DocIndex, false);
+ _lastState.DocIndex.Write(output, absolute);
+ if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
+ {
+ _lastState.FreqIndex.CopyFrom(state.FreqIndex, false);
+ _lastState.FreqIndex.Write(output, absolute);
+ if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ _lastState.PosIndex.CopyFrom(state.PosIndex, false);
+ _lastState.PosIndex.Write(output, absolute);
+ if (STORE_PAYLOADS)
+ {
+ if (absolute)
+ {
+ output.WriteVLong(state.PayloadFp);
+ }
+ else
+ {
+ output.WriteVLong(state.PayloadFp - LAST_PAYLOAD_FP);
+ }
+ LAST_PAYLOAD_FP = state.PayloadFp;
+ }
+ }
+ }
+ if (state.SkipFp == -1) return;
+
+ if (absolute)
+ {
+ output.WriteVLong(state.SkipFp);
+ }
+ else
+ {
+ output.WriteVLong(state.SkipFp - LAST_SKIP_FP);
+ }
+ LAST_SKIP_FP = state.SkipFp;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing) return;
+
+ IOUtils.Close(DOC_OUT, SKIP_OUT, FREQ_OUT, POS_OUT, PAYLOAD_OUT);
}
- }
- }
- if (state.skipFP != -1) {
- if (absolute) {
- out.writeVLong(state.skipFP);
- } else {
- out.writeVLong(state.skipFP - lastSkipFP);
- }
- lastSkipFP = state.skipFP;
}
- }
- @Override
- public void close() {
- IOUtils.close(docOut, skipOut, freqOut, posOut, payloadOut);
- }
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs b/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs
index aee9180..b194b67 100644
--- a/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs
+++ b/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,193 +15,202 @@ package codecs.sep;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Arrays;
-
-import store.IndexInput;
-import codecs.MultiLevelSkipListReader;
-import index.FieldInfo.IndexOptions;
-
-/**
- * Implements the skip list reader for the default posting list format
- * that stores positions and payloads.
- *
- * @lucene.experimental
- */
-
-// TODO: rewrite this as recursive classes?
-class SepSkipListReader extends MultiLevelSkipListReader {
- private bool currentFieldStoresPayloads;
- private IntIndexInput.Index freqIndex[];
- private IntIndexInput.Index docIndex[];
- private IntIndexInput.Index posIndex[];
- private long payloadPointer[];
- private int payloadLength[];
-
- private final IntIndexInput.Index lastFreqIndex;
- private final IntIndexInput.Index lastDocIndex;
- // TODO: -- make private again
- final IntIndexInput.Index lastPosIndex;
-
- private long lastPayloadPointer;
- private int lastPayloadLength;
-
- SepSkipListReader(IndexInput skipStream,
- IntIndexInput freqIn,
- IntIndexInput docIn,
- IntIndexInput posIn,
- int maxSkipLevels,
- int skipInterval)
- {
- super(skipStream, maxSkipLevels, skipInterval);
- if (freqIn != null) {
- freqIndex = new IntIndexInput.Index[maxSkipLevels];
- }
- docIndex = new IntIndexInput.Index[maxSkipLevels];
- if (posIn != null) {
- posIndex = new IntIndexInput.Index[maxNumberOfSkipLevels];
- }
- for(int i=0;i<maxSkipLevels;i++) {
- if (freqIn != null) {
- freqIndex[i] = freqIn.index();
- }
- docIndex[i] = docIn.index();
- if (posIn != null) {
- posIndex[i] = posIn.index();
- }
- }
- payloadPointer = new long[maxSkipLevels];
- payloadLength = new int[maxSkipLevels];
-
- if (freqIn != null) {
- lastFreqIndex = freqIn.index();
- } else {
- lastFreqIndex = null;
- }
- lastDocIndex = docIn.index();
- if (posIn != null) {
- lastPosIndex = posIn.index();
- } else {
- lastPosIndex = null;
- }
- }
-
- IndexOptions indexOptions;
-
- void setIndexOptions(IndexOptions v) {
- indexOptions = v;
- }
-
- void init(long skipPointer,
- IntIndexInput.Index docBaseIndex,
- IntIndexInput.Index freqBaseIndex,
- IntIndexInput.Index posBaseIndex,
- long payloadBasePointer,
- int df,
- bool storesPayloads) {
-
- super.init(skipPointer, df);
- this.currentFieldStoresPayloads = storesPayloads;
-
- lastPayloadPointer = payloadBasePointer;
-
- for(int i=0;i<maxNumberOfSkipLevels;i++) {
- docIndex[i].copyFrom(docBaseIndex);
- if (freqIndex != null) {
- freqIndex[i].copyFrom(freqBaseIndex);
- }
- if (posBaseIndex != null) {
- posIndex[i].copyFrom(posBaseIndex);
- }
- }
- Arrays.fill(payloadPointer, payloadBasePointer);
- Arrays.fill(payloadLength, 0);
- }
-
- long getPayloadPointer() {
- return lastPayloadPointer;
- }
-
- /** Returns the payload length of the payload stored just before
- * the doc to which the last call of {@link MultiLevelSkipListReader#skipTo(int)}
- * has skipped. */
- int getPayloadLength() {
- return lastPayloadLength;
- }
-
- @Override
- protected void seekChild(int level) {
- super.seekChild(level);
- payloadPointer[level] = lastPayloadPointer;
- payloadLength[level] = lastPayloadLength;
- }
-
- @Override
- protected void setLastSkipData(int level) {
- super.setLastSkipData(level);
-
- lastPayloadPointer = payloadPointer[level];
- lastPayloadLength = payloadLength[level];
- if (freqIndex != null) {
- lastFreqIndex.copyFrom(freqIndex[level]);
- }
- lastDocIndex.copyFrom(docIndex[level]);
- if (lastPosIndex != null) {
- lastPosIndex.copyFrom(posIndex[level]);
+namespace Lucene.Net.Codecs.Sep
+{
+ using System.Diagnostics;
+ using Index;
+ using Store;
+ using Support;
+
+ /// <summary>
+ /// Implements the skip list reader for the default posting list format
+ /// that stores positions and payloads.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+
+ // TODO: rewrite this as recursive classes?
+ internal class SepSkipListReader : MultiLevelSkipListReader
+ {
+ private bool _currentFieldStoresPayloads;
+ private readonly IntIndexInputIndex[] _freqIndex;
+ private readonly IntIndexInputIndex[] _docIndex;
+ private readonly IntIndexInputIndex[] _posIndex;
+ private readonly long[] _payloadPointer;
+ private readonly int[] _payloadLength;
+
+ private readonly IntIndexInputIndex _lastFreqIndex;
+ private readonly IntIndexInputIndex _lastDocIndex;
+ private readonly IntIndexInputIndex _lastPosIndex;
+
+ private FieldInfo.IndexOptions _indexOptions;
+
+ private long _lastPayloadPointer;
+ private int _lastPayloadLength;
+
+ internal SepSkipListReader(IndexInput skipStream, IntIndexInput freqIn, IntIndexInput docIn, IntIndexInput posIn,
+ int maxSkipLevels, int skipInterval)
+ : base(skipStream, maxSkipLevels, skipInterval)
+ {
+ if (freqIn != null)
+ _freqIndex = new IntIndexInputIndex[maxSkipLevels];
+
+ _docIndex = new IntIndexInputIndex[maxSkipLevels];
+
+ if (posIn != null)
+ _posIndex = new IntIndexInputIndex[MaxNumberOfSkipLevels];
+
+ for (var i = 0; i < maxSkipLevels; i++)
+ {
+ if (freqIn != null)
+ _freqIndex[i] = freqIn.Index();
+
+ _docIndex[i] = docIn.Index();
+
+ if (posIn != null)
+ _posIndex[i] = posIn.Index();
+ }
+
+ _payloadPointer = new long[maxSkipLevels];
+ _payloadLength = new int[maxSkipLevels];
+
+ _lastFreqIndex = freqIn != null ? freqIn.Index() : null;
+ _lastDocIndex = docIn.Index();
+ _lastPosIndex = posIn != null ? posIn.Index() : null;
+ }
+
+
+ internal virtual FieldInfo.IndexOptions IndexOptions
+ {
+ set { _indexOptions = value; }
+ }
+
+ internal virtual void Init(long skipPointer, IntIndexInputIndex docBaseIndex, IntIndexInputIndex freqBaseIndex,
+ IntIndexInputIndex posBaseIndex, long payloadBasePointer, int df, bool storesPayloads)
+ {
+
+ base.Init(skipPointer, df);
+ _currentFieldStoresPayloads = storesPayloads;
+
+ _lastPayloadPointer = payloadBasePointer;
+
+ for (var i = 0; i < MaxNumberOfSkipLevels; i++)
+ {
+ _docIndex[i].CopyFrom(docBaseIndex);
+ if (_freqIndex != null)
+ _freqIndex[i].CopyFrom(freqBaseIndex);
+
+ if (posBaseIndex != null)
+ _posIndex[i].CopyFrom(posBaseIndex);
+ }
+ Arrays.Fill(_payloadPointer, payloadBasePointer);
+ Arrays.Fill(_payloadLength, 0);
+ }
+
+ internal virtual long PayloadPointer
+ {
+ get { return _lastPayloadPointer; }
+ }
+
+ /// <summary>
+ /// Returns the payload length of the payload stored just before
+ /// the doc to which the last call of <seealso cref="MultiLevelSkipListReader#skipTo(int)"/>
+ /// has skipped.
+ /// </summary>
+ internal virtual int PayloadLength
+ {
+ get { return _lastPayloadLength; }
+ }
+
+ protected override void SeekChild(int level)
+ {
+ base.SeekChild(level);
+ _payloadPointer[level] = _lastPayloadPointer;
+ _payloadLength[level] = _lastPayloadLength;
+ }
+
+ protected override int LastSkipData
+ {
+ set
+ {
+ base.LastSkipData = value;
+
+ _lastPayloadPointer = _payloadPointer[value];
+ _lastPayloadLength = _payloadLength[value];
+
+ if (_freqIndex != null)
+ _lastFreqIndex.CopyFrom(_freqIndex[value]);
+
+ _lastDocIndex.CopyFrom(_docIndex[value]);
+
+ if (_lastPosIndex != null)
+ _lastPosIndex.CopyFrom(_posIndex[value]);
+
+ if (value <= 0) return;
+
+ if (_freqIndex != null)
+ _freqIndex[value - 1].CopyFrom(_freqIndex[value]);
+
+ _docIndex[value - 1].CopyFrom(_docIndex[value]);
+
+ if (_posIndex != null)
+ _posIndex[value - 1].CopyFrom(_posIndex[value]);
+
+ }
+ }
+
+ internal virtual IntIndexInputIndex FreqIndex
+ {
+ get { return _lastFreqIndex; }
+ }
+
+ internal virtual IntIndexInputIndex PosIndex
+ {
+ get { return _lastPosIndex; }
+ }
+
+ internal virtual IntIndexInputIndex DocIndex
+ {
+ get { return _lastDocIndex; }
+ }
+
+ protected override int ReadSkipData(int level, IndexInput skipStream)
+ {
+ int delta;
+ Debug.Assert(_indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS ||
+ !_currentFieldStoresPayloads);
+ if (_currentFieldStoresPayloads)
+ {
+ // the current field stores payloads.
+ // if the doc delta is odd then we have
+ // to read the current payload length
+ // because it differs from the length of the
+ // previous payload
+ delta = skipStream.ReadVInt();
+ if ((delta & 1) != 0)
+ {
+ _payloadLength[level] = skipStream.ReadVInt();
+ }
+ delta = (int) ((uint) delta >> 1);
+ }
+ else
+ {
+ delta = skipStream.ReadVInt();
+ }
+
+ if (_indexOptions != FieldInfo.IndexOptions.DOCS_ONLY)
+ _freqIndex[level].Read(skipStream, false);
+
+ _docIndex[level].Read(skipStream, false);
+ if (_indexOptions != FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) return delta;
+
+ _posIndex[level].Read(skipStream, false);
+
+ if (_currentFieldStoresPayloads)
+ _payloadPointer[level] += skipStream.ReadVInt();
+
+ return delta;
+ }
}
- if (level > 0) {
- if (freqIndex != null) {
- freqIndex[level-1].copyFrom(freqIndex[level]);
- }
- docIndex[level-1].copyFrom(docIndex[level]);
- if (posIndex != null) {
- posIndex[level-1].copyFrom(posIndex[level]);
- }
- }
- }
-
- IntIndexInput.Index getFreqIndex() {
- return lastFreqIndex;
- }
-
- IntIndexInput.Index getPosIndex() {
- return lastPosIndex;
- }
-
- IntIndexInput.Index getDocIndex() {
- return lastDocIndex;
- }
-
- @Override
- protected int readSkipData(int level, IndexInput skipStream) {
- int delta;
- Debug.Assert( indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !currentFieldStoresPayloads;
- if (currentFieldStoresPayloads) {
- // the current field stores payloads.
- // if the doc delta is odd then we have
- // to read the current payload length
- // because it differs from the length of the
- // previous payload
- delta = skipStream.readVInt();
- if ((delta & 1) != 0) {
- payloadLength[level] = skipStream.readVInt();
- }
- delta >>>= 1;
- } else {
- delta = skipStream.readVInt();
- }
- if (indexOptions != IndexOptions.DOCS_ONLY) {
- freqIndex[level].read(skipStream, false);
- }
- docIndex[level].read(skipStream, false);
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- posIndex[level].read(skipStream, false);
- if (currentFieldStoresPayloads) {
- payloadPointer[level] += skipStream.readVInt();
- }
- }
-
- return delta;
- }
-}
+}
\ No newline at end of file
[04/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
index 015a1af..5ab0b09 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
@@ -1,658 +1,995 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import codecs.CodecUtil;
-import codecs.DocValuesProducer;
-import index.BinaryDocValues;
-import index.CorruptIndexException;
-import index.DocValues;
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.NumericDocValues;
-import index.SegmentReadState;
-import index.SortedDocValues;
-import index.SortedSetDocValues;
-import index.TermsEnum;
-import store.ByteArrayDataInput;
-import store.ChecksumIndexInput;
-import store.IndexInput;
-import util.Bits;
-import util.BytesRef;
-import util.FixedBitSet;
-import util.IOUtils;
-import util.IntsRef;
-import util.PagedBytes;
-import util.RamUsageEstimator;
-import util.fst.BytesRefFSTEnum;
-import util.fst.BytesRefFSTEnum.InputOutput;
-import util.fst.FST;
-import util.fst.FST.Arc;
-import util.fst.FST.BytesReader;
-import util.fst.PositiveIntOutputs;
-import util.fst.Util;
-import util.packed.BlockPackedReader;
-import util.packed.MonotonicBlockPackedReader;
-import util.packed.PackedInts;
-
-/**
- * Reader for {@link MemoryDocValuesFormat}
- */
-class MemoryDocValuesProducer extends DocValuesProducer {
- // metadata maps (just file pointers and minimal stuff)
- private final Map<Integer,NumericEntry> numerics;
- private final Map<Integer,BinaryEntry> binaries;
- private final Map<Integer,FSTEntry> fsts;
- private final IndexInput data;
-
- // ram instances we have already loaded
- private final Map<Integer,NumericDocValues> numericInstances =
- new HashMap<>();
- private final Map<Integer,BinaryDocValues> binaryInstances =
- new HashMap<>();
- private final Map<Integer,FST<Long>> fstInstances =
- new HashMap<>();
- private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<>();
-
- private final int maxDoc;
- private final AtomicLong ramBytesUsed;
- private final int version;
-
- static final byte NUMBER = 0;
- static final byte BYTES = 1;
- static final byte FST = 2;
-
- static final int BLOCK_SIZE = 4096;
-
- static final byte DELTA_COMPRESSED = 0;
- static final byte TABLE_COMPRESSED = 1;
- static final byte UNCOMPRESSED = 2;
- static final byte GCD_COMPRESSED = 3;
-
- static final int VERSION_START = 0;
- static final int VERSION_GCD_COMPRESSION = 1;
- static final int VERSION_CHECKSUM = 2;
- static final int VERSION_CURRENT = VERSION_CHECKSUM;
-
- MemoryDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) {
- maxDoc = state.segmentInfo.getDocCount();
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- // read in the entries from the metadata file.
- ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
- bool success = false;
- try {
- version = CodecUtil.checkHeader(in, metaCodec,
- VERSION_START,
- VERSION_CURRENT);
- numerics = new HashMap<>();
- binaries = new HashMap<>();
- fsts = new HashMap<>();
- readFields(in, state.fieldInfos);
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checkFooter(in);
- } else {
- CodecUtil.checkEOF(in);
- }
- ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
- }
- }
-
- success = false;
- try {
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = state.directory.openInput(dataName, state.context);
- final int version2 = CodecUtil.checkHeader(data, dataCodec,
- VERSION_START,
- VERSION_CURRENT);
- if (version != version2) {
- throw new CorruptIndexException("Format versions mismatch");
- }
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this.data);
- }
- }
- }
-
- private void readFields(IndexInput meta, FieldInfos infos) {
- int fieldNumber = meta.readVInt();
- while (fieldNumber != -1) {
- int fieldType = meta.readByte();
- if (fieldType == NUMBER) {
- NumericEntry entry = new NumericEntry();
- entry.offset = meta.readLong();
- entry.missingOffset = meta.readLong();
- if (entry.missingOffset != -1) {
- entry.missingBytes = meta.readLong();
- } else {
- entry.missingBytes = 0;
- }
- entry.format = meta.readByte();
- switch(entry.format) {
- case DELTA_COMPRESSED:
- case TABLE_COMPRESSED:
- case GCD_COMPRESSED:
- case UNCOMPRESSED:
- break;
- default:
- throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
- }
- if (entry.format != UNCOMPRESSED) {
- entry.packedIntsVersion = meta.readVInt();
- }
- numerics.put(fieldNumber, entry);
- } else if (fieldType == BYTES) {
- BinaryEntry entry = new BinaryEntry();
- entry.offset = meta.readLong();
- entry.numBytes = meta.readLong();
- entry.missingOffset = meta.readLong();
- if (entry.missingOffset != -1) {
- entry.missingBytes = meta.readLong();
- } else {
- entry.missingBytes = 0;
- }
- entry.minLength = meta.readVInt();
- entry.maxLength = meta.readVInt();
- if (entry.minLength != entry.maxLength) {
- entry.packedIntsVersion = meta.readVInt();
- entry.blockSize = meta.readVInt();
- }
- binaries.put(fieldNumber, entry);
- } else if (fieldType == FST) {
- FSTEntry entry = new FSTEntry();
- entry.offset = meta.readLong();
- entry.numOrds = meta.readVLong();
- fsts.put(fieldNumber, entry);
- } else {
- throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
- }
- fieldNumber = meta.readVInt();
- }
- }
-
- @Override
- public synchronized NumericDocValues getNumeric(FieldInfo field) {
- NumericDocValues instance = numericInstances.get(field.number);
- if (instance == null) {
- instance = loadNumeric(field);
- numericInstances.put(field.number, instance);
- }
- return instance;
- }
-
- @Override
- public long ramBytesUsed() {
- return ramBytesUsed.get();
- }
-
- @Override
- public void checkIntegrity() {
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(data);
- }
- }
-
- private NumericDocValues loadNumeric(FieldInfo field) {
- NumericEntry entry = numerics.get(field.number);
- data.seek(entry.offset + entry.missingBytes);
- switch (entry.format) {
- case TABLE_COMPRESSED:
- int size = data.readVInt();
- if (size > 256) {
- throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + data);
- }
- final long decode[] = new long[size];
- for (int i = 0; i < decode.length; i++) {
- decode[i] = data.readLong();
- }
- final int formatID = data.readVInt();
- final int bitsPerValue = data.readVInt();
- final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
- return new NumericDocValues() {
- @Override
- public long get(int docID) {
- return decode[(int)ordsReader.get(docID)];
- }
- };
- case DELTA_COMPRESSED:
- final int blockSize = data.readVInt();
- final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
- ramBytesUsed.addAndGet(reader.ramBytesUsed());
- return reader;
- case UNCOMPRESSED:
- final byte bytes[] = new byte[maxDoc];
- data.readBytes(bytes, 0, bytes.length);
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
- return new NumericDocValues() {
- @Override
- public long get(int docID) {
- return bytes[docID];
- }
- };
- case GCD_COMPRESSED:
- final long min = data.readLong();
- final long mult = data.readLong();
- final int quotientBlockSize = data.readVInt();
- final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
- ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
- return new NumericDocValues() {
- @Override
- public long get(int docID) {
- return min + mult * quotientReader.get(docID);
- }
- };
- default:
- throw new Debug.Assert(ionError();
- }
- }
-
- @Override
- public synchronized BinaryDocValues getBinary(FieldInfo field) {
- BinaryDocValues instance = binaryInstances.get(field.number);
- if (instance == null) {
- instance = loadBinary(field);
- binaryInstances.put(field.number, instance);
- }
- return instance;
- }
-
- private BinaryDocValues loadBinary(FieldInfo field) {
- BinaryEntry entry = binaries.get(field.number);
- data.seek(entry.offset);
- PagedBytes bytes = new PagedBytes(16);
- bytes.copy(data, entry.numBytes);
- final PagedBytes.Reader bytesReader = bytes.freeze(true);
- if (entry.minLength == entry.maxLength) {
- final int fixedLength = entry.minLength;
- ramBytesUsed.addAndGet(bytes.ramBytesUsed());
- return new BinaryDocValues() {
- @Override
- public void get(int docID, BytesRef result) {
- bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
- }
- };
- } else {
- data.seek(data.getFilePointer() + entry.missingBytes);
- final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
- return new BinaryDocValues() {
- @Override
- public void get(int docID, BytesRef result) {
- long startAddress = docID == 0 ? 0 : addresses.get(docID-1);
- long endAddress = addresses.get(docID);
- bytesReader.fillSlice(result, startAddress, (int) (endAddress - startAddress));
- }
- };
- }
- }
-
- @Override
- public SortedDocValues getSorted(FieldInfo field) {
- final FSTEntry entry = fsts.get(field.number);
- if (entry.numOrds == 0) {
- return DocValues.EMPTY_SORTED;
- }
- FST<Long> instance;
- synchronized(this) {
- instance = fstInstances.get(field.number);
- if (instance == null) {
- data.seek(entry.offset);
- instance = new FST<>(data, PositiveIntOutputs.getSingleton());
- ramBytesUsed.addAndGet(instance.sizeInBytes());
- fstInstances.put(field.number, instance);
- }
- }
- final NumericDocValues docToOrd = getNumeric(field);
- final FST<Long> fst = instance;
-
- // per-thread resources
- final BytesReader in = fst.getBytesReader();
- final Arc<Long> firstArc = new Arc<>();
- final Arc<Long> scratchArc = new Arc<>();
- final IntsRef scratchInts = new IntsRef();
- final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
-
- return new SortedDocValues() {
- @Override
- public int getOrd(int docID) {
- return (int) docToOrd.get(docID);
- }
-
- @Override
- public void lookupOrd(int ord, BytesRef result) {
- try {
- in.setPosition(0);
- fst.getFirstArc(firstArc);
- IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
- result.bytes = new byte[output.length];
- result.offset = 0;
- result.length = 0;
- Util.toBytesRef(output, result);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public int lookupTerm(BytesRef key) {
- try {
- InputOutput<Long> o = fstEnum.seekCeil(key);
- if (o == null) {
- return -getValueCount()-1;
- } else if (o.input.equals(key)) {
- return o.output.intValue();
- } else {
- return (int) -o.output-1;
- }
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public int getValueCount() {
- return (int)entry.numOrds;
- }
-
- @Override
- public TermsEnum termsEnum() {
- return new FSTTermsEnum(fst);
- }
- };
- }
-
- @Override
- public SortedSetDocValues getSortedSet(FieldInfo field) {
- final FSTEntry entry = fsts.get(field.number);
- if (entry.numOrds == 0) {
- return DocValues.EMPTY_SORTED_SET; // empty FST!
- }
- FST<Long> instance;
- synchronized(this) {
- instance = fstInstances.get(field.number);
- if (instance == null) {
- data.seek(entry.offset);
- instance = new FST<>(data, PositiveIntOutputs.getSingleton());
- ramBytesUsed.addAndGet(instance.sizeInBytes());
- fstInstances.put(field.number, instance);
- }
- }
- final BinaryDocValues docToOrds = getBinary(field);
- final FST<Long> fst = instance;
-
- // per-thread resources
- final BytesReader in = fst.getBytesReader();
- final Arc<Long> firstArc = new Arc<>();
- final Arc<Long> scratchArc = new Arc<>();
- final IntsRef scratchInts = new IntsRef();
- final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
- final BytesRef ref = new BytesRef();
- final ByteArrayDataInput input = new ByteArrayDataInput();
- return new SortedSetDocValues() {
- long currentOrd;
-
- @Override
- public long nextOrd() {
- if (input.eof()) {
- return NO_MORE_ORDS;
- } else {
- currentOrd += input.readVLong();
- return currentOrd;
- }
- }
-
- @Override
- public void setDocument(int docID) {
- docToOrds.get(docID, ref);
- input.reset(ref.bytes, ref.offset, ref.length);
- currentOrd = 0;
- }
-
- @Override
- public void lookupOrd(long ord, BytesRef result) {
- try {
- in.setPosition(0);
- fst.getFirstArc(firstArc);
- IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
- result.bytes = new byte[output.length];
- result.offset = 0;
- result.length = 0;
- Util.toBytesRef(output, result);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public long lookupTerm(BytesRef key) {
- try {
- InputOutput<Long> o = fstEnum.seekCeil(key);
- if (o == null) {
- return -getValueCount()-1;
- } else if (o.input.equals(key)) {
- return o.output.intValue();
- } else {
- return -o.output-1;
- }
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public long getValueCount() {
- return entry.numOrds;
- }
-
- @Override
- public TermsEnum termsEnum() {
- return new FSTTermsEnum(fst);
- }
- };
- }
-
- private Bits getMissingBits(int fieldNumber, final long offset, final long length) {
- if (offset == -1) {
- return new Bits.MatchAllBits(maxDoc);
- } else {
- Bits instance;
- synchronized(this) {
- instance = docsWithFieldInstances.get(fieldNumber);
- if (instance == null) {
- IndexInput data = this.data.clone();
- data.seek(offset);
- Debug.Assert( length % 8 == 0;
- long bits[] = new long[(int) length >> 3];
- for (int i = 0; i < bits.length; i++) {
- bits[i] = data.readLong();
- }
- instance = new FixedBitSet(bits, maxDoc);
- docsWithFieldInstances.put(fieldNumber, instance);
- }
- }
- return instance;
- }
- }
-
- @Override
- public Bits getDocsWithField(FieldInfo field) {
- switch(field.getDocValuesType()) {
- case SORTED_SET:
- return DocValues.docsWithValue(getSortedSet(field), maxDoc);
- case SORTED:
- return DocValues.docsWithValue(getSorted(field), maxDoc);
- case BINARY:
- BinaryEntry be = binaries.get(field.number);
- return getMissingBits(field.number, be.missingOffset, be.missingBytes);
- case NUMERIC:
- NumericEntry ne = numerics.get(field.number);
- return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
- default:
- throw new Debug.Assert(ionError();
- }
- }
-
- @Override
- public void close() {
- data.close();
- }
-
- static class NumericEntry {
- long offset;
- long missingOffset;
- long missingBytes;
- byte format;
- int packedIntsVersion;
- }
-
- static class BinaryEntry {
- long offset;
- long missingOffset;
- long missingBytes;
- long numBytes;
- int minLength;
- int maxLength;
- int packedIntsVersion;
- int blockSize;
- }
-
- static class FSTEntry {
- long offset;
- long numOrds;
- }
-
- // exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
- static class FSTTermsEnum extends TermsEnum {
- final BytesRefFSTEnum<Long> in;
-
- // this is all for the complicated seek(ord)...
- // maybe we should add a FSTEnum that supports this operation?
- final FST<Long> fst;
- final FST.BytesReader bytesReader;
- final Arc<Long> firstArc = new Arc<>();
- final Arc<Long> scratchArc = new Arc<>();
- final IntsRef scratchInts = new IntsRef();
- final BytesRef scratchBytes = new BytesRef();
-
- FSTTermsEnum(FST<Long> fst) {
- this.fst = fst;
- in = new BytesRefFSTEnum<>(fst);
- bytesReader = fst.getBytesReader();
- }
-
- @Override
- public BytesRef next() {
- InputOutput<Long> io = in.next();
- if (io == null) {
- return null;
- } else {
- return io.input;
- }
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef text) {
- if (in.seekCeil(text) == null) {
- return SeekStatus.END;
- } else if (term().equals(text)) {
- // TODO: add SeekStatus to FSTEnum like in https://issues.apache.org/jira/browse/LUCENE-3729
- // to remove this comparision?
- return SeekStatus.FOUND;
- } else {
- return SeekStatus.NOT_FOUND;
- }
- }
-
- @Override
- public bool seekExact(BytesRef text) {
- if (in.seekExact(text) == null) {
- return false;
- } else {
- return true;
- }
- }
-
- @Override
- public void seekExact(long ord) {
- // TODO: would be better to make this simpler and faster.
- // but we dont want to introduce a bug that corrupts our enum state!
- bytesReader.setPosition(0);
- fst.getFirstArc(firstArc);
- IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
- scratchBytes.bytes = new byte[output.length];
- scratchBytes.offset = 0;
- scratchBytes.length = 0;
- Util.toBytesRef(output, scratchBytes);
- // TODO: we could do this lazily, better to try to push into FSTEnum though?
- in.seekExact(scratchBytes);
- }
-
- @Override
- public BytesRef term() {
- return in.current().input;
- }
-
- @Override
- public long ord() {
- return in.current().output;
- }
-
- @Override
- public int docFreq() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long totalTermFreq() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
- throw new UnsupportedOperationException();
- }
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Lucene.Net.Codecs.Sep;
+using org.apache.lucene.codecs.memory;
+
+namespace Lucene.Net.Codecs.Memory
+{
+ /// <summary>
+ /// Reader for <seealso cref="MemoryDocValuesFormat"/>
+ /// </summary>
+ internal class MemoryDocValuesProducer : DocValuesProducer
+ {
+ // metadata maps (just file pointers and minimal stuff)
+ private readonly IDictionary<int?, NumericEntry> numerics;
+ private readonly IDictionary<int?, BinaryEntry> binaries;
+ private readonly IDictionary<int?, FSTEntry> fsts;
+ private readonly IndexInput data;
+
+ // ram instances we have already loaded
+ private readonly IDictionary<int?, NumericDocValues> numericInstances = new Dictionary<int?, NumericDocValues>();
+ private readonly IDictionary<int?, BinaryDocValues> binaryInstances = new Dictionary<int?, BinaryDocValues>();
+ private readonly IDictionary<int?, FST<long?>> fstInstances = new Dictionary<int?, FST<long?>>();
+ private readonly IDictionary<int?, Bits> docsWithFieldInstances = new Dictionary<int?, Bits>();
+
+ private readonly int maxDoc;
+ private readonly AtomicLong ramBytesUsed_Renamed;
+ private readonly int version;
+
+ internal const sbyte NUMBER = 0;
+ internal const sbyte BYTES = 1;
+ internal const sbyte org;
+
+ internal const int BLOCK_SIZE = 4096;
+
+ internal const sbyte DELTA_COMPRESSED = 0;
+ internal const sbyte TABLE_COMPRESSED = 1;
+ internal const sbyte UNCOMPRESSED = 2;
+ internal const sbyte GCD_COMPRESSED = 3;
+
+ internal const int VERSION_START = 0;
+ internal const int VERSION_GCD_COMPRESSION = 1;
+ internal const int VERSION_CHECKSUM = 2;
+ internal const int VERSION_CURRENT = VERSION_CHECKSUM;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: MemoryDocValuesProducer(org.apache.lucene.index.SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws java.io.IOException
+ internal MemoryDocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
+ {
+ maxDoc = state.segmentInfo.DocCount;
+ string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+ // read in the entries from the metadata file.
+ ChecksumIndexInput @in = state.directory.openChecksumInput(metaName, state.context);
+ bool success = false;
+ try
+ {
+ version = CodecUtil.checkHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
+ numerics = new Dictionary<>();
+ binaries = new Dictionary<>();
+ fsts = new Dictionary<>();
+ readFields(@in, state.fieldInfos);
+ if (version >= VERSION_CHECKSUM)
+ {
+ CodecUtil.checkFooter(@in);
+ }
+ else
+ {
+ CodecUtil.checkEOF(@in);
+ }
+ ramBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(this.GetType()));
+ success = true;
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.close(@in);
+ }
+ else
+ {
+ IOUtils.closeWhileHandlingException(@in);
+ }
+ }
+
+ success = false;
+ try
+ {
+ string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+ data = state.directory.openInput(dataName, state.context);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int version2 = org.apache.lucene.codecs.CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
+ int version2 = CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
+ if (version != version2)
+ {
+ throw new CorruptIndexException("Format versions mismatch");
+ }
+
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(this.data);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void readFields(org.apache.lucene.store.IndexInput meta, org.apache.lucene.index.FieldInfos infos) throws java.io.IOException
+ private void readFields(IndexInput meta, FieldInfos infos)
+ {
+ int fieldNumber = meta.readVInt();
+ while (fieldNumber != -1)
+ {
+ int fieldType = meta.readByte();
+ if (fieldType == NUMBER)
+ {
+ NumericEntry entry = new NumericEntry();
+ entry.offset = meta.readLong();
+ entry.missingOffset = meta.readLong();
+ if (entry.missingOffset != -1)
+ {
+ entry.missingBytes = meta.readLong();
+ }
+ else
+ {
+ entry.missingBytes = 0;
+ }
+ entry.format = meta.readByte();
+ switch (entry.format)
+ {
+ case DELTA_COMPRESSED:
+ case TABLE_COMPRESSED:
+ case GCD_COMPRESSED:
+ case UNCOMPRESSED:
+ break;
+ default:
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ if (entry.format != UNCOMPRESSED)
+ {
+ entry.packedIntsVersion = meta.readVInt();
+ }
+ numerics[fieldNumber] = entry;
+ }
+ else if (fieldType == BYTES)
+ {
+ BinaryEntry entry = new BinaryEntry();
+ entry.offset = meta.readLong();
+ entry.numBytes = meta.readLong();
+ entry.missingOffset = meta.readLong();
+ if (entry.missingOffset != -1)
+ {
+ entry.missingBytes = meta.readLong();
+ }
+ else
+ {
+ entry.missingBytes = 0;
+ }
+ entry.minLength = meta.readVInt();
+ entry.maxLength = meta.readVInt();
+ if (entry.minLength != entry.maxLength)
+ {
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ }
+ binaries[fieldNumber] = entry;
+ }
+ else if (fieldType == FST)
+ {
+ FSTEntry entry = new FSTEntry();
+ entry.offset = meta.readLong();
+ entry.numOrds = meta.readVLong();
+ fsts[fieldNumber] = entry;
+ }
+ else
+ {
+ throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
+ }
+ fieldNumber = meta.readVInt();
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.NumericDocValues getNumeric(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override NumericDocValues getNumeric(FieldInfo field)
+ {
+ lock (this)
+ {
+ NumericDocValues instance = numericInstances[field.number];
+ if (instance == null)
+ {
+ instance = loadNumeric(field);
+ numericInstances[field.number] = instance;
+ }
+ return instance;
+ }
+ }
+
+ public override long ramBytesUsed()
+ {
+ return ramBytesUsed_Renamed.get();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+ public override void checkIntegrity()
+ {
+ if (version >= VERSION_CHECKSUM)
+ {
+ CodecUtil.checksumEntireFile(data);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.NumericDocValues loadNumeric(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ private NumericDocValues loadNumeric(FieldInfo field)
+ {
+ NumericEntry entry = numerics[field.number];
+ data.seek(entry.offset + entry.missingBytes);
+ switch (entry.format)
+ {
+ case TABLE_COMPRESSED:
+ int size = data.readVInt();
+ if (size > 256)
+ {
+ throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + data);
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long decode[] = new long[size];
+ long[] decode = new long[size];
+ for (int i = 0; i < decode.Length; i++)
+ {
+ decode[i] = data.readLong();
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int formatID = data.readVInt();
+ int formatID = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int bitsPerValue = data.readVInt();
+ int bitsPerValue = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.PackedInts.Reader ordsReader = org.apache.lucene.util.packed.PackedInts.getReaderNoHeader(data, org.apache.lucene.util.packed.PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
+ PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
+ return new NumericDocValuesAnonymousInnerClassHelper(this, decode, ordsReader);
+ case DELTA_COMPRESSED:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int blockSize = data.readVInt();
+ int blockSize = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.BlockPackedReader reader = new org.apache.lucene.util.packed.BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
+ BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
+ ramBytesUsed_Renamed.addAndGet(reader.ramBytesUsed());
+ return reader;
+ case UNCOMPRESSED:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final byte bytes[] = new byte[maxDoc];
+ sbyte[] bytes = new sbyte[maxDoc];
+ data.readBytes(bytes, 0, bytes.Length);
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes));
+ return new NumericDocValuesAnonymousInnerClassHelper2(this, bytes);
+ case GCD_COMPRESSED:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long min = data.readLong();
+ long min = data.readLong();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long mult = data.readLong();
+ long mult = data.readLong();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int quotientBlockSize = data.readVInt();
+ int quotientBlockSize = data.readVInt();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.BlockPackedReader quotientReader = new org.apache.lucene.util.packed.BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
+ BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
+ ramBytesUsed_Renamed.addAndGet(quotientReader.ramBytesUsed());
+ return new NumericDocValuesAnonymousInnerClassHelper3(this, min, mult, quotientReader);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper : NumericDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private long[] decode;
+ private IntIndexInput.Reader ordsReader;
+
+ public NumericDocValuesAnonymousInnerClassHelper(MemoryDocValuesProducer outerInstance, long[] decode, IntIndexInput.Reader ordsReader)
+ {
+ this.outerInstance = outerInstance;
+ this.decode = decode;
+ this.ordsReader = ordsReader;
+ }
+
+ public override long get(int docID)
+ {
+ return decode[(int)ordsReader.get(docID)];
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper2 : NumericDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private sbyte[] bytes;
+
+ public NumericDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, sbyte[] bytes)
+ {
+ this.outerInstance = outerInstance;
+ this.bytes = bytes;
+ }
+
+ public override long get(int docID)
+ {
+ return bytes[docID];
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper3 : NumericDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private long min;
+ private long mult;
+ private BlockPackedReader quotientReader;
+
+ public NumericDocValuesAnonymousInnerClassHelper3(MemoryDocValuesProducer outerInstance, long min, long mult, BlockPackedReader quotientReader)
+ {
+ this.outerInstance = outerInstance;
+ this.min = min;
+ this.mult = mult;
+ this.quotientReader = quotientReader;
+ }
+
+ public override long get(int docID)
+ {
+ return min + mult * quotientReader.get(docID);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.BinaryDocValues getBinary(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override BinaryDocValues getBinary(FieldInfo field)
+ {
+ lock (this)
+ {
+ BinaryDocValues instance = binaryInstances[field.number];
+ if (instance == null)
+ {
+ instance = loadBinary(field);
+ binaryInstances[field.number] = instance;
+ }
+ return instance;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.BinaryDocValues loadBinary(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ private BinaryDocValues loadBinary(FieldInfo field)
+ {
+ BinaryEntry entry = binaries[field.number];
+ data.seek(entry.offset);
+ PagedBytes bytes = new PagedBytes(16);
+ bytes.copy(data, entry.numBytes);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.PagedBytes.Reader bytesReader = bytes.freeze(true);
+ PagedBytes.Reader bytesReader = bytes.freeze(true);
+ if (entry.minLength == entry.maxLength)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int fixedLength = entry.minLength;
+ int fixedLength = entry.minLength;
+ ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed());
+ return new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength);
+ }
+ else
+ {
+ data.seek(data.FilePointer + entry.missingBytes);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.packed.MonotonicBlockPackedReader addresses = new org.apache.lucene.util.packed.MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
+ MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
+ ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
+ return new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses);
+ }
+ }
+
+ private class BinaryDocValuesAnonymousInnerClassHelper : BinaryDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private IntIndexInput.Reader bytesReader;
+ private int fixedLength;
+
+ public BinaryDocValuesAnonymousInnerClassHelper(MemoryDocValuesProducer outerInstance, IntIndexInput.Reader bytesReader, int fixedLength)
+ {
+ this.outerInstance = outerInstance;
+ this.bytesReader = bytesReader;
+ this.fixedLength = fixedLength;
+ }
+
+ public override void get(int docID, BytesRef result)
+ {
+ bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
+ }
+ }
+
+ private class BinaryDocValuesAnonymousInnerClassHelper2 : BinaryDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private IntIndexInput.Reader bytesReader;
+ private MonotonicBlockPackedReader addresses;
+
+ public BinaryDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, IntIndexInput.Reader bytesReader, MonotonicBlockPackedReader addresses)
+ {
+ this.outerInstance = outerInstance;
+ this.bytesReader = bytesReader;
+ this.addresses = addresses;
+ }
+
+ public override void get(int docID, BytesRef result)
+ {
+ long startAddress = docID == 0 ? 0 : addresses.get(docID - 1);
+ long endAddress = addresses.get(docID);
+ bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.SortedDocValues getSorted(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override SortedDocValues getSorted(FieldInfo field)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final FSTEntry entry = fsts.get(field.number);
+ FSTEntry entry = fsts[field.number];
+ if (entry.numOrds == 0)
+ {
+ return DocValues.EMPTY_SORTED;
+ }
+ FST<long?> instance;
+ lock (this)
+ {
+ instance = fstInstances[field.number];
+ if (instance == null)
+ {
+ data.seek(entry.offset);
+ instance = new FST<>(data, PositiveIntOutputs.Singleton);
+ ramBytesUsed_Renamed.addAndGet(instance.sizeInBytes());
+ fstInstances[field.number] = instance;
+ }
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.NumericDocValues docToOrd = getNumeric(field);
+ NumericDocValues docToOrd = getNumeric(field);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST<Long> fst = instance;
+ FST<long?> fst = instance;
+
+ // per-thread resources
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.BytesReader in = fst.getBytesReader();
+ FST.BytesReader @in = fst.BytesReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> firstArc = new org.apache.lucene.util.fst.FST.Arc<>();
+ FST.Arc<long?> firstArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> scratchArc = new org.apache.lucene.util.fst.FST.Arc<>();
+ FST.Arc<long?> scratchArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.IntsRef scratchInts = new org.apache.lucene.util.IntsRef();
+ IntsRef scratchInts = new IntsRef();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.BytesRefFSTEnum<Long> fstEnum = new org.apache.lucene.util.fst.BytesRefFSTEnum<>(fst);
+ BytesRefFSTEnum<long?> fstEnum = new BytesRefFSTEnum<long?>(fst);
+
+ return new SortedDocValuesAnonymousInnerClassHelper(this, entry, docToOrd, fst, @in, firstArc, scratchArc, scratchInts, fstEnum);
+ }
+
+ private class SortedDocValuesAnonymousInnerClassHelper : SortedDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private MemoryDocValuesProducer.FSTEntry entry;
+ private NumericDocValues docToOrd;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST<long?> fst;
+ private FST<long?> fst;
+ private FST.BytesReader @in;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> firstArc;
+ private FST.Arc<long?> firstArc;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> scratchArc;
+ private FST.Arc<long?> scratchArc;
+ private IntsRef scratchInts;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.BytesRefFSTEnum<long?> fstEnum;
+ private BytesRefFSTEnum<long?> fstEnum;
+
+ public SortedDocValuesAnonymousInnerClassHelper<T1, T2, T3, T4>(MemoryDocValuesProducer outerInstance, org.MemoryDocValuesProducer.FSTEntry entry, NumericDocValues docToOrd, FST<T1> fst, FST.BytesReader @in, FST.Arc<T2> firstArc, FST.Arc<T3> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<T4> fstEnum)
+ {
+ this.outerInstance = outerInstance;
+ this.entry = entry;
+ this.docToOrd = docToOrd;
+ this.fst = fst;
+ this.@in = @in;
+ this.firstArc = firstArc;
+ this.scratchArc = scratchArc;
+ this.scratchInts = scratchInts;
+ this.fstEnum = fstEnum;
+ }
+
+ public override int getOrd(int docID)
+ {
+ return (int) docToOrd.get(docID);
+ }
+
+ public override void lookupOrd(int ord, BytesRef result)
+ {
+ try
+ {
+ @in.Position = 0;
+ fst.getFirstArc(firstArc);
+ IntsRef output = Util.getByOutput(fst, ord, @in, firstArc, scratchArc, scratchInts);
+ result.bytes = new sbyte[output.length];
+ result.offset = 0;
+ result.length = 0;
+ Util.toBytesRef(output, result);
+ }
+ catch (IOException bogus)
+ {
+ throw new Exception(bogus);
+ }
+ }
+
+ public override int lookupTerm(BytesRef key)
+ {
+ try
+ {
+ BytesRefFSTEnum.InputOutput<long?> o = fstEnum.seekCeil(key);
+ if (o == null)
+ {
+ return -ValueCount - 1;
+ }
+ else if (o.input.Equals(key))
+ {
+ return (int)o.output;
+ }
+ else
+ {
+ return (int) -o.output - 1;
+ }
+ }
+ catch (IOException bogus)
+ {
+ throw new Exception(bogus);
+ }
+ }
+
+ public override int ValueCount
+ {
+ get
+ {
+ return (int)entry.numOrds;
+ }
+ }
+
+ public override TermsEnum termsEnum()
+ {
+ return new FSTTermsEnum(fst);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.SortedSetDocValues getSortedSet(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override SortedSetDocValues getSortedSet(FieldInfo field)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final FSTEntry entry = fsts.get(field.number);
+ FSTEntry entry = fsts[field.number];
+ if (entry.numOrds == 0)
+ {
+ return DocValues.EMPTY_SORTED_SET; // empty FST!
+ }
+ FST<long?> instance;
+ lock (this)
+ {
+ instance = fstInstances[field.number];
+ if (instance == null)
+ {
+ data.seek(entry.offset);
+ instance = new FST<>(data, PositiveIntOutputs.Singleton);
+ ramBytesUsed_Renamed.addAndGet(instance.sizeInBytes());
+ fstInstances[field.number] = instance;
+ }
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.BinaryDocValues docToOrds = getBinary(field);
+ BinaryDocValues docToOrds = getBinary(field);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST<Long> fst = instance;
+ FST<long?> fst = instance;
+
+ // per-thread resources
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.BytesReader in = fst.getBytesReader();
+ FST.BytesReader @in = fst.BytesReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> firstArc = new org.apache.lucene.util.fst.FST.Arc<>();
+ FST.Arc<long?> firstArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> scratchArc = new org.apache.lucene.util.fst.FST.Arc<>();
+ FST.Arc<long?> scratchArc = new FST.Arc<long?>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.IntsRef scratchInts = new org.apache.lucene.util.IntsRef();
+ IntsRef scratchInts = new IntsRef();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.BytesRefFSTEnum<Long> fstEnum = new org.apache.lucene.util.fst.BytesRefFSTEnum<>(fst);
+ BytesRefFSTEnum<long?> fstEnum = new BytesRefFSTEnum<long?>(fst);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.BytesRef ref = new org.apache.lucene.util.BytesRef();
+ BytesRef @ref = new BytesRef();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.store.ByteArrayDataInput input = new org.apache.lucene.store.ByteArrayDataInput();
+ ByteArrayDataInput input = new ByteArrayDataInput();
+ return new SortedSetDocValuesAnonymousInnerClassHelper(this, entry, docToOrds, fst, @in, firstArc, scratchArc, scratchInts, fstEnum, @ref, input);
+ }
+
+ private class SortedSetDocValuesAnonymousInnerClassHelper : SortedSetDocValues
+ {
+ private readonly MemoryDocValuesProducer outerInstance;
+
+ private MemoryDocValuesProducer.FSTEntry entry;
+ private BinaryDocValues docToOrds;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST<long?> fst;
+ private FST<long?> fst;
+ private FST.BytesReader @in;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> firstArc;
+ private FST.Arc<long?> firstArc;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.FST.Arc<long?> scratchArc;
+ private FST.Arc<long?> scratchArc;
+ private IntsRef scratchInts;
+//JAVA TO C# CONVERTER TODO TASK: Java wildcard generics are not converted to .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.fst.BytesRefFSTEnum<long?> fstEnum;
+ private BytesRefFSTEnum<long?> fstEnum;
+ private BytesRef @ref;
+ private ByteArrayDataInput input;
+
+ public SortedSetDocValuesAnonymousInnerClassHelper<T1, T2, T3, T4>(MemoryDocValuesProducer outerInstance, org.MemoryDocValuesProducer.FSTEntry entry, BinaryDocValues docToOrds, FST<T1> fst, FST.BytesReader @in, FST.Arc<T2> firstArc, FST.Arc<T3> scratchArc, IntsRef scratchInts, BytesRefFSTEnum<T4> fstEnum, BytesRef @ref, ByteArrayDataInput input)
+ {
+ this.outerInstance = outerInstance;
+ this.entry = entry;
+ this.docToOrds = docToOrds;
+ this.fst = fst;
+ this.@in = @in;
+ this.firstArc = firstArc;
+ this.scratchArc = scratchArc;
+ this.scratchInts = scratchInts;
+ this.fstEnum = fstEnum;
+ this.@ref = @ref;
+ this.input = input;
+ }
+
+ internal long currentOrd;
+
+ public override long nextOrd()
+ {
+ if (input.eof())
+ {
+ return NO_MORE_ORDS;
+ }
+ else
+ {
+ currentOrd += input.readVLong();
+ return currentOrd;
+ }
+ }
+
+ public override int Document
+ {
+ set
+ {
+ docToOrds.get(value, @ref);
+ input.reset(@ref.bytes, @ref.offset, @ref.length);
+ currentOrd = 0;
+ }
+ }
+
+ public override void lookupOrd(long ord, BytesRef result)
+ {
+ try
+ {
+ @in.Position = 0;
+ fst.getFirstArc(firstArc);
+ IntsRef output = Util.getByOutput(fst, ord, @in, firstArc, scratchArc, scratchInts);
+ result.bytes = new sbyte[output.length];
+ result.offset = 0;
+ result.length = 0;
+ Util.toBytesRef(output, result);
+ }
+ catch (IOException bogus)
+ {
+ throw new Exception(bogus);
+ }
+ }
+
+ public override long lookupTerm(BytesRef key)
+ {
+ try
+ {
+ BytesRefFSTEnum.InputOutput<long?> o = fstEnum.seekCeil(key);
+ if (o == null)
+ {
+ return -ValueCount - 1;
+ }
+ else if (o.input.Equals(key))
+ {
+ return (int)o.output;
+ }
+ else
+ {
+ return -o.output - 1;
+ }
+ }
+ catch (IOException bogus)
+ {
+ throw new Exception(bogus);
+ }
+ }
+
+ public override long ValueCount
+ {
+ get
+ {
+ return entry.numOrds;
+ }
+ }
+
+ public override TermsEnum termsEnum()
+ {
+ return new FSTTermsEnum(fst);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.Bits getMissingBits(int fieldNumber, final long offset, final long length) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ private Bits getMissingBits(int fieldNumber, long offset, long length)
+ {
+ if (offset == -1)
+ {
+ return new Bits.MatchAllBits(maxDoc);
+ }
+ else
+ {
+ Bits instance;
+ lock (this)
+ {
+ instance = docsWithFieldInstances[fieldNumber];
+ if (instance == null)
+ {
+ IndexInput data = this.data.clone();
+ data.seek(offset);
+ Debug.Assert(length % 8 == 0);
+ long[] bits = new long[(int) length >> 3];
+ for (int i = 0; i < bits.Length; i++)
+ {
+ bits[i] = data.readLong();
+ }
+ instance = new FixedBitSet(bits, maxDoc);
+ docsWithFieldInstances[fieldNumber] = instance;
+ }
+ }
+ return instance;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.Bits getDocsWithField(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override Bits getDocsWithField(FieldInfo field)
+ {
+ switch (field.DocValuesType)
+ {
+ case SORTED_SET:
+ return DocValues.docsWithValue(getSortedSet(field), maxDoc);
+ case SORTED:
+ return DocValues.docsWithValue(getSorted(field), maxDoc);
+ case BINARY:
+ BinaryEntry be = binaries[field.number];
+ return getMissingBits(field.number, be.missingOffset, be.missingBytes);
+ case NUMERIC:
+ NumericEntry ne = numerics[field.number];
+ return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ data.close();
+ }
+
+ internal class NumericEntry
+ {
+ internal long offset;
+ internal long missingOffset;
+ internal long missingBytes;
+ internal sbyte format;
+ internal int packedIntsVersion;
+ }
+
+ internal class BinaryEntry
+ {
+ internal long offset;
+ internal long missingOffset;
+ internal long missingBytes;
+ internal long numBytes;
+ internal int minLength;
+ internal int maxLength;
+ internal int packedIntsVersion;
+ internal int blockSize;
+ }
+
+ internal class FSTEntry
+ {
+ internal long offset;
+ internal long numOrds;
+ }
+
+ // exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
+ internal class FSTTermsEnum : TermsEnum
+ {
+ internal readonly BytesRefFSTEnum<long?> @in;
+
+ // this is all for the complicated seek(ord)...
+ // maybe we should add a FSTEnum that supports this operation?
+ internal readonly FST<long?> fst;
+ internal readonly FST.BytesReader bytesReader;
+ internal readonly FST.Arc<long?> firstArc = new FST.Arc<long?>();
+ internal readonly FST.Arc<long?> scratchArc = new FST.Arc<long?>();
+ internal readonly IntsRef scratchInts = new IntsRef();
+ internal readonly BytesRef scratchBytes = new BytesRef();
+
+ internal FSTTermsEnum(FST<long?> fst)
+ {
+ this.fst = fst;
+ @in = new BytesRefFSTEnum<>(fst);
+ bytesReader = fst.BytesReader;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+ public override BytesRef next()
+ {
+ BytesRefFSTEnum.InputOutput<long?> io = @in.next();
+ if (io == null)
+ {
+ return null;
+ }
+ else
+ {
+ return io.input;
+ }
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+ public override SeekStatus seekCeil(BytesRef text)
+ {
+ if (@in.seekCeil(text) == null)
+ {
+ return SeekStatus.END;
+ }
+ else if (term().Equals(text))
+ {
+ // TODO: add SeekStatus to FSTEnum like in https://issues.apache.org/jira/browse/LUCENE-3729
+ // to remove this comparision?
+ return SeekStatus.FOUND;
+ }
+ else
+ {
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean seekExact(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+ public override bool seekExact(BytesRef text)
+ {
+ if (@in.seekExact(text) == null)
+ {
+ return false;
+ }
+ else
+ {
+ return true;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void seekExact(long ord) throws java.io.IOException
+ public override void seekExact(long ord)
+ {
+ // TODO: would be better to make this simpler and faster.
+ // but we dont want to introduce a bug that corrupts our enum state!
+ bytesReader.Position = 0;
+ fst.getFirstArc(firstArc);
+ IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
+ scratchBytes.bytes = new sbyte[output.length];
+ scratchBytes.offset = 0;
+ scratchBytes.length = 0;
+ Util.toBytesRef(output, scratchBytes);
+ // TODO: we could do this lazily, better to try to push into FSTEnum though?
+ @in.seekExact(scratchBytes);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef term() throws java.io.IOException
+ public override BytesRef term()
+ {
+ return @in.current().input;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long ord() throws java.io.IOException
+ public override long ord()
+ {
+ return @in.current().output;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int docFreq() throws java.io.IOException
+ public override int docFreq()
+ {
+ throw new System.NotSupportedException();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long totalTermFreq() throws java.io.IOException
+ public override long totalTermFreq()
+ {
+ throw new System.NotSupportedException();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsEnum docs(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsEnum reuse, int flags) throws java.io.IOException
+ public override DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
+ {
+ throw new System.NotSupportedException();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsAndPositionsEnum docsAndPositions(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsAndPositionsEnum reuse, int flags) throws java.io.IOException
+ public override DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags)
+ {
+ throw new System.NotSupportedException();
+ }
+ }
+ }
+
+}
\ No newline at end of file
[09/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
index c28a147..310a771 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
@@ -1,2302 +1,2936 @@
-package codecs.memory;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-
-import codecs.FieldsConsumer;
-import codecs.FieldsProducer;
-import codecs.PostingsFormat;
-import codecs.lucene41.Lucene41PostingsFormat; // javadocs
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.Fields;
-import index.OrdTermState;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import index.TermState;
-import index.Terms;
-import index.TermsEnum;
-import store.IOContext;
-import store.RAMOutputStream;
-import util.ArrayUtil;
-import util.Bits;
-import util.BytesRef;
-import util.RamUsageEstimator;
-import util.automaton.CompiledAutomaton;
-import util.automaton.RunAutomaton;
-import util.automaton.Transition;
-
-// TODO:
-// - build depth-N prefix hash?
-// - or: longer dense skip lists than just next byte?
-
-/** Wraps {@link Lucene41PostingsFormat} format for on-disk
- * storage, but then at read time loads and stores all
- * terms & postings directly in RAM as byte[], int[].
- *
- * <p><b><font color=red>WARNING</font></b>: This is
- * exceptionally RAM intensive: it makes no effort to
- * compress the postings data, storing terms as separate
- * byte[] and postings as separate int[], but as a result it
- * gives substantial increase in search performance.
- *
- * <p>This postings format supports {@link TermsEnum#ord}
- * and {@link TermsEnum#seekExact(long)}.
-
- * <p>Because this holds all term bytes as a single
- * byte[], you cannot have more than 2.1GB worth of term
- * bytes in a single segment.
- *
- * @lucene.experimental */
-
-public final class DirectPostingsFormat extends PostingsFormat {
-
- private final int minSkipCount;
- private final int lowFreqCutoff;
-
- private final static int DEFAULT_MIN_SKIP_COUNT = 8;
- private final static int DEFAULT_LOW_FREQ_CUTOFF = 32;
-
- //private static final bool DEBUG = true;
-
- // TODO: allow passing/wrapping arbitrary postings format?
-
- public DirectPostingsFormat() {
- this(DEFAULT_MIN_SKIP_COUNT, DEFAULT_LOW_FREQ_CUTOFF);
- }
-
- /** minSkipCount is how many terms in a row must have the
- * same prefix before we put a skip pointer down. Terms
- * with docFreq <= lowFreqCutoff will use a single int[]
- * to hold all docs, freqs, position and offsets; terms
- * with higher docFreq will use separate arrays. */
- public DirectPostingsFormat(int minSkipCount, int lowFreqCutoff) {
- super("Direct");
- this.minSkipCount = minSkipCount;
- this.lowFreqCutoff = lowFreqCutoff;
- }
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
- return PostingsFormat.forName("Lucene41").fieldsConsumer(state);
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) {
- FieldsProducer postings = PostingsFormat.forName("Lucene41").fieldsProducer(state);
- if (state.context.context != IOContext.Context.MERGE) {
- FieldsProducer loadedPostings;
- try {
- postings.checkIntegrity();
- loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff);
- } finally {
- postings.close();
- }
- return loadedPostings;
- } else {
- // Don't load postings for merge:
- return postings;
- }
- }
-
- private static final class DirectFields extends FieldsProducer {
- private final Map<String,DirectField> fields = new TreeMap<>();
-
- public DirectFields(SegmentReadState state, Fields fields, int minSkipCount, int lowFreqCutoff) {
- for (String field : fields) {
- this.fields.put(field, new DirectField(state, field, fields.terms(field), minSkipCount, lowFreqCutoff));
- }
- }
-
- @Override
- public Iterator<String> iterator() {
- return Collections.unmodifiableSet(fields.keySet()).iterator();
- }
-
- @Override
- public Terms terms(String field) {
- return fields.get(field);
- }
-
- @Override
- public int size() {
- return fields.size();
- }
-
- @Override
- public long getUniqueTermCount() {
- long numTerms = 0;
- for(DirectField field : fields.values()) {
- numTerms += field.terms.length;
- }
- return numTerms;
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public long ramBytesUsed() {
- long sizeInBytes = 0;
- for(Map.Entry<String,DirectField> entry: fields.entrySet()) {
- sizeInBytes += entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR;
- sizeInBytes += entry.getValue().ramBytesUsed();
- }
- return sizeInBytes;
- }
-
- @Override
- public void checkIntegrity() {
- // if we read entirely into ram, we already validated.
- // otherwise returned the raw postings reader
- }
- }
-
- private final static class DirectField extends Terms {
-
- private static abstract class TermAndSkip {
- public int[] skips;
-
- /** Returns the approximate number of RAM bytes used */
- public abstract long ramBytesUsed();
- }
-
- private static final class LowFreqTerm extends TermAndSkip {
- public final int[] postings;
- public final byte[] payloads;
- public final int docFreq;
- public final int totalTermFreq;
-
- public LowFreqTerm(int[] postings, byte[] payloads, int docFreq, int totalTermFreq) {
- this.postings = postings;
- this.payloads = payloads;
- this.docFreq = docFreq;
- this.totalTermFreq = totalTermFreq;
- }
-
- @Override
- public long ramBytesUsed() {
- return ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) +
- ((payloads!=null) ? RamUsageEstimator.sizeOf(payloads) : 0);
- }
- }
-
- // TODO: maybe specialize into prx/no-prx/no-frq cases?
- private static final class HighFreqTerm extends TermAndSkip {
- public final long totalTermFreq;
- public final int[] docIDs;
- public final int[] freqs;
- public final int[][] positions;
- public final byte[][][] payloads;
-
- public HighFreqTerm(int[] docIDs, int[] freqs, int[][] positions, byte[][][] payloads, long totalTermFreq) {
- this.docIDs = docIDs;
- this.freqs = freqs;
- this.positions = positions;
- this.payloads = payloads;
- this.totalTermFreq = totalTermFreq;
- }
-
- @Override
- public long ramBytesUsed() {
- long sizeInBytes = 0;
- sizeInBytes += (docIDs!=null)? RamUsageEstimator.sizeOf(docIDs) : 0;
- sizeInBytes += (freqs!=null)? RamUsageEstimator.sizeOf(freqs) : 0;
-
- if(positions != null) {
- for(int[] position : positions) {
- sizeInBytes += (position!=null) ? RamUsageEstimator.sizeOf(position) : 0;
- }
- }
-
- if (payloads != null) {
- for(byte[][] payload : payloads) {
- if(payload != null) {
- for(byte[] pload : payload) {
- sizeInBytes += (pload!=null) ? RamUsageEstimator.sizeOf(pload) : 0;
- }
- }
- }
- }
-
- return sizeInBytes;
- }
- }
-
- private final byte[] termBytes;
- private final int[] termOffsets;
-
- private final int[] skips;
- private final int[] skipOffsets;
-
- private final TermAndSkip[] terms;
- private final bool hasFreq;
- private final bool hasPos;
- private final bool hasOffsets;
- private final bool hasPayloads;
- private final long sumTotalTermFreq;
- private final int docCount;
- private final long sumDocFreq;
- private int skipCount;
-
- // TODO: maybe make a separate builder? These are only
- // used during load:
- private int count;
- private int[] sameCounts = new int[10];
- private final int minSkipCount;
-
- private final static class IntArrayWriter {
- private int[] ints = new int[10];
- private int upto;
-
- public void add(int value) {
- if (ints.length == upto) {
- ints = ArrayUtil.grow(ints);
- }
- ints[upto++] = value;
- }
-
- public int[] get() {
- final int[] arr = new int[upto];
- System.arraycopy(ints, 0, arr, 0, upto);
- upto = 0;
- return arr;
- }
- }
-
- public DirectField(SegmentReadState state, String field, Terms termsIn, int minSkipCount, int lowFreqCutoff) {
- final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
-
- sumTotalTermFreq = termsIn.getSumTotalTermFreq();
- sumDocFreq = termsIn.getSumDocFreq();
- docCount = termsIn.getDocCount();
-
- final int numTerms = (int) termsIn.size();
- if (numTerms == -1) {
- throw new IllegalArgumentException("codec does not provide Terms.size()");
- }
- terms = new TermAndSkip[numTerms];
- termOffsets = new int[1+numTerms];
-
- byte[] termBytes = new byte[1024];
-
- this.minSkipCount = minSkipCount;
-
- hasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_ONLY) > 0;
- hasPos = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) > 0;
- hasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) > 0;
- hasPayloads = fieldInfo.hasPayloads();
-
- BytesRef term;
- DocsEnum docsEnum = null;
- DocsAndPositionsEnum docsAndPositionsEnum = null;
- final TermsEnum termsEnum = termsIn.iterator(null);
- int termOffset = 0;
-
- final IntArrayWriter scratch = new IntArrayWriter();
-
- // Used for payloads, if any:
- final RAMOutputStream ros = new RAMOutputStream();
-
- // if (DEBUG) {
- // System.out.println("\nLOAD terms seg=" + state.segmentInfo.name + " field=" + field + " hasOffsets=" + hasOffsets + " hasFreq=" + hasFreq + " hasPos=" + hasPos + " hasPayloads=" + hasPayloads);
- // }
-
- while ((term = termsEnum.next()) != null) {
- final int docFreq = termsEnum.docFreq();
- final long totalTermFreq = termsEnum.totalTermFreq();
-
- // if (DEBUG) {
- // System.out.println(" term=" + term.utf8ToString());
- // }
-
- termOffsets[count] = termOffset;
-
- if (termBytes.length < (termOffset + term.length)) {
- termBytes = ArrayUtil.grow(termBytes, termOffset + term.length);
- }
- System.arraycopy(term.bytes, term.offset, termBytes, termOffset, term.length);
- termOffset += term.length;
- termOffsets[count+1] = termOffset;
-
- if (hasPos) {
- docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum);
- } else {
- docsEnum = termsEnum.docs(null, docsEnum);
- }
-
- final TermAndSkip ent;
-
- final DocsEnum docsEnum2;
- if (hasPos) {
- docsEnum2 = docsAndPositionsEnum;
- } else {
- docsEnum2 = docsEnum;
- }
-
- int docID;
-
- if (docFreq <= lowFreqCutoff) {
-
- ros.reset();
-
- // Pack postings for low-freq terms into a single int[]:
- while ((docID = docsEnum2.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
- scratch.add(docID);
- if (hasFreq) {
- final int freq = docsEnum2.freq();
- scratch.add(freq);
- if (hasPos) {
- for(int pos=0;pos<freq;pos++) {
- scratch.add(docsAndPositionsEnum.nextPosition());
- if (hasOffsets) {
- scratch.add(docsAndPositionsEnum.startOffset());
- scratch.add(docsAndPositionsEnum.endOffset());
- }
- if (hasPayloads) {
- final BytesRef payload = docsAndPositionsEnum.getPayload();
- if (payload != null) {
- scratch.add(payload.length);
- ros.writeBytes(payload.bytes, payload.offset, payload.length);
- } else {
- scratch.add(0);
- }
- }
- }
- }
- }
- }
-
- final byte[] payloads;
- if (hasPayloads) {
- ros.flush();
- payloads = new byte[(int) ros.length()];
- ros.writeTo(payloads, 0);
- } else {
- payloads = null;
- }
-
- final int[] postings = scratch.get();
-
- ent = new LowFreqTerm(postings, payloads, docFreq, (int) totalTermFreq);
- } else {
- final int[] docs = new int[docFreq];
- final int[] freqs;
- final int[][] positions;
- final byte[][][] payloads;
- if (hasFreq) {
- freqs = new int[docFreq];
- if (hasPos) {
- positions = new int[docFreq][];
- if (hasPayloads) {
- payloads = new byte[docFreq][][];
- } else {
- payloads = null;
- }
- } else {
- positions = null;
- payloads = null;
- }
- } else {
- freqs = null;
- positions = null;
- payloads = null;
- }
-
- // Use separate int[] for the postings for high-freq
- // terms:
- int upto = 0;
- while ((docID = docsEnum2.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
- docs[upto] = docID;
- if (hasFreq) {
- final int freq = docsEnum2.freq();
- freqs[upto] = freq;
- if (hasPos) {
- final int mult;
- if (hasOffsets) {
- mult = 3;
- } else {
- mult = 1;
- }
- if (hasPayloads) {
- payloads[upto] = new byte[freq][];
- }
- positions[upto] = new int[mult*freq];
- int posUpto = 0;
- for(int pos=0;pos<freq;pos++) {
- positions[upto][posUpto] = docsAndPositionsEnum.nextPosition();
- if (hasPayloads) {
- BytesRef payload = docsAndPositionsEnum.getPayload();
- if (payload != null) {
- byte[] payloadBytes = new byte[payload.length];
- System.arraycopy(payload.bytes, payload.offset, payloadBytes, 0, payload.length);
- payloads[upto][pos] = payloadBytes;
- }
- }
- posUpto++;
- if (hasOffsets) {
- positions[upto][posUpto++] = docsAndPositionsEnum.startOffset();
- positions[upto][posUpto++] = docsAndPositionsEnum.endOffset();
- }
- }
- }
- }
-
- upto++;
- }
- Debug.Assert( upto == docFreq;
- ent = new HighFreqTerm(docs, freqs, positions, payloads, totalTermFreq);
- }
-
- terms[count] = ent;
- setSkips(count, termBytes);
- count++;
- }
-
- // End sentinel:
- termOffsets[count] = termOffset;
-
- finishSkips();
-
- //System.out.println(skipCount + " skips: " + field);
-
- this.termBytes = new byte[termOffset];
- System.arraycopy(termBytes, 0, this.termBytes, 0, termOffset);
-
- // Pack skips:
- this.skips = new int[skipCount];
- this.skipOffsets = new int[1+numTerms];
-
- int skipOffset = 0;
- for(int i=0;i<numTerms;i++) {
- final int[] termSkips = terms[i].skips;
- skipOffsets[i] = skipOffset;
- if (termSkips != null) {
- System.arraycopy(termSkips, 0, skips, skipOffset, termSkips.length);
- skipOffset += termSkips.length;
- terms[i].skips = null;
- }
- }
- this.skipOffsets[numTerms] = skipOffset;
- Debug.Assert( skipOffset == skipCount;
- }
-
- /** Returns approximate RAM bytes used */
- public long ramBytesUsed() {
- long sizeInBytes = 0;
- sizeInBytes += ((termBytes!=null) ? RamUsageEstimator.sizeOf(termBytes) : 0);
- sizeInBytes += ((termOffsets!=null) ? RamUsageEstimator.sizeOf(termOffsets) : 0);
- sizeInBytes += ((skips!=null) ? RamUsageEstimator.sizeOf(skips) : 0);
- sizeInBytes += ((skipOffsets!=null) ? RamUsageEstimator.sizeOf(skipOffsets) : 0);
- sizeInBytes += ((sameCounts!=null) ? RamUsageEstimator.sizeOf(sameCounts) : 0);
-
- if(terms!=null) {
- for(TermAndSkip termAndSkip : terms) {
- sizeInBytes += (termAndSkip!=null) ? termAndSkip.ramBytesUsed() : 0;
- }
- }
-
- return sizeInBytes;
- }
-
- // Compares in unicode (UTF8) order:
- int compare(int ord, BytesRef other) {
- final byte[] otherBytes = other.bytes;
-
- int upto = termOffsets[ord];
- final int termLen = termOffsets[1+ord] - upto;
- int otherUpto = other.offset;
-
- final int stop = upto + Math.min(termLen, other.length);
- while (upto < stop) {
- int diff = (termBytes[upto++] & 0xFF) - (otherBytes[otherUpto++] & 0xFF);
- if (diff != 0) {
- return diff;
- }
- }
-
- // One is a prefix of the other, or, they are equal:
- return termLen - other.length;
- }
-
- private void setSkips(int termOrd, byte[] termBytes) {
-
- final int termLength = termOffsets[termOrd+1] - termOffsets[termOrd];
-
- if (sameCounts.length < termLength) {
- sameCounts = ArrayUtil.grow(sameCounts, termLength);
- }
-
- // Update skip pointers:
- if (termOrd > 0) {
- final int lastTermLength = termOffsets[termOrd] - termOffsets[termOrd-1];
- final int limit = Math.min(termLength, lastTermLength);
-
- int lastTermOffset = termOffsets[termOrd-1];
- int termOffset = termOffsets[termOrd];
-
- int i = 0;
- for(;i<limit;i++) {
- if (termBytes[lastTermOffset++] == termBytes[termOffset++]) {
- sameCounts[i]++;
- } else {
- for(;i<limit;i++) {
- if (sameCounts[i] >= minSkipCount) {
- // Go back and add a skip pointer:
- saveSkip(termOrd, sameCounts[i]);
- }
- sameCounts[i] = 1;
- }
- break;
- }
- }
-
- for(;i<lastTermLength;i++) {
- if (sameCounts[i] >= minSkipCount) {
- // Go back and add a skip pointer:
- saveSkip(termOrd, sameCounts[i]);
- }
- sameCounts[i] = 0;
- }
- for(int j=limit;j<termLength;j++) {
- sameCounts[j] = 1;
- }
- } else {
- for(int i=0;i<termLength;i++) {
- sameCounts[i]++;
- }
- }
- }
-
- private void finishSkips() {
- Debug.Assert( count == terms.length;
- int lastTermOffset = termOffsets[count-1];
- int lastTermLength = termOffsets[count] - lastTermOffset;
-
- for(int i=0;i<lastTermLength;i++) {
- if (sameCounts[i] >= minSkipCount) {
- // Go back and add a skip pointer:
- saveSkip(count, sameCounts[i]);
- }
- }
-
- // Reverse the skip pointers so they are "nested":
- for(int termID=0;termID<terms.length;termID++) {
- TermAndSkip term = terms[termID];
- if (term.skips != null && term.skips.length > 1) {
- for(int pos=0;pos<term.skips.length/2;pos++) {
- final int otherPos = term.skips.length-pos-1;
-
- final int temp = term.skips[pos];
- term.skips[pos] = term.skips[otherPos];
- term.skips[otherPos] = temp;
- }
- }
- }
- }
-
- private void saveSkip(int ord, int backCount) {
- final TermAndSkip term = terms[ord - backCount];
- skipCount++;
- if (term.skips == null) {
- term.skips = new int[] {ord};
- } else {
- // Normally we'd grow at a slight exponential... but
- // given that the skips themselves are already log(N)
- // we can grow by only 1 and still have amortized
- // linear time:
- final int[] newSkips = new int[term.skips.length+1];
- System.arraycopy(term.skips, 0, newSkips, 0, term.skips.length);
- term.skips = newSkips;
- term.skips[term.skips.length-1] = ord;
- }
- }
-
- @Override
- public TermsEnum iterator(TermsEnum reuse) {
- DirectTermsEnum termsEnum;
- if (reuse != null && reuse instanceof DirectTermsEnum) {
- termsEnum = (DirectTermsEnum) reuse;
- if (!termsEnum.canReuse(terms)) {
- termsEnum = new DirectTermsEnum();
- }
- } else {
- termsEnum = new DirectTermsEnum();
- }
- termsEnum.reset();
- return termsEnum;
- }
-
- @Override
- public TermsEnum intersect(CompiledAutomaton compiled, final BytesRef startTerm) {
- return new DirectIntersectTermsEnum(compiled, startTerm);
- }
-
- @Override
- public long size() {
- return terms.length;
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return sumTotalTermFreq;
- }
-
- @Override
- public long getSumDocFreq() {
- return sumDocFreq;
- }
-
- @Override
- public int getDocCount() {
- return docCount;
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public bool hasFreqs() {
- return hasFreq;
- }
-
- @Override
- public bool hasOffsets() {
- return hasOffsets;
- }
-
- @Override
- public bool hasPositions() {
- return hasPos;
- }
-
- @Override
- public bool hasPayloads() {
- return hasPayloads;
- }
-
- private final class DirectTermsEnum extends TermsEnum {
-
- private final BytesRef scratch = new BytesRef();
- private int termOrd;
-
- bool canReuse(TermAndSkip[] other) {
- return DirectField.this.terms == other;
- }
-
- private BytesRef setTerm() {
- scratch.bytes = termBytes;
- scratch.offset = termOffsets[termOrd];
- scratch.length = termOffsets[termOrd+1] - termOffsets[termOrd];
- return scratch;
- }
-
- public void reset() {
- termOrd = -1;
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public BytesRef next() {
- termOrd++;
- if (termOrd < terms.length) {
- return setTerm();
- } else {
- return null;
- }
- }
-
- @Override
- public TermState termState() {
- OrdTermState state = new OrdTermState();
- state.ord = termOrd;
- return state;
- }
-
- // If non-negative, exact match; else, -ord-1, where ord
- // is where you would insert the term.
- private int findTerm(BytesRef term) {
-
- // Just do binary search: should be (constant factor)
- // faster than using the skip list:
- int low = 0;
- int high = terms.length-1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- int cmp = compare(mid, term);
- if (cmp < 0) {
- low = mid + 1;
- } else if (cmp > 0) {
- high = mid - 1;
- } else {
- return mid; // key found
- }
- }
-
- return -(low + 1); // key not found.
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef term) {
- // TODO: we should use the skip pointers; should be
- // faster than bin search; we should also hold
- // & reuse current state so seeking forwards is
- // faster
- final int ord = findTerm(term);
- // if (DEBUG) {
- // System.out.println(" find term=" + term.utf8ToString() + " ord=" + ord);
- // }
- if (ord >= 0) {
- termOrd = ord;
- setTerm();
- return SeekStatus.FOUND;
- } else if (ord == -terms.length-1) {
- return SeekStatus.END;
- } else {
- termOrd = -ord - 1;
- setTerm();
- return SeekStatus.NOT_FOUND;
- }
- }
-
- @Override
- public bool seekExact(BytesRef term) {
- // TODO: we should use the skip pointers; should be
- // faster than bin search; we should also hold
- // & reuse current state so seeking forwards is
- // faster
- final int ord = findTerm(term);
- if (ord >= 0) {
- termOrd = ord;
- setTerm();
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void seekExact(long ord) {
- termOrd = (int) ord;
- setTerm();
- }
-
- @Override
- public void seekExact(BytesRef term, TermState state) {
- termOrd = (int) ((OrdTermState) state).ord;
- setTerm();
- Debug.Assert( term.equals(scratch);
- }
-
- @Override
- public BytesRef term() {
- return scratch;
- }
-
- @Override
- public long ord() {
- return termOrd;
- }
-
- @Override
- public int docFreq() {
- if (terms[termOrd] instanceof LowFreqTerm) {
- return ((LowFreqTerm) terms[termOrd]).docFreq;
- } else {
- return ((HighFreqTerm) terms[termOrd]).docIDs.length;
- }
- }
-
- @Override
- public long totalTermFreq() {
- if (terms[termOrd] instanceof LowFreqTerm) {
- return ((LowFreqTerm) terms[termOrd]).totalTermFreq;
- } else {
- return ((HighFreqTerm) terms[termOrd]).totalTermFreq;
- }
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
- // TODO: implement reuse, something like Pulsing:
- // it's hairy!
-
- if (terms[termOrd] instanceof LowFreqTerm) {
- final int[] postings = ((LowFreqTerm) terms[termOrd]).postings;
- if (hasFreq) {
- if (hasPos) {
- int posLen;
- if (hasOffsets) {
- posLen = 3;
- } else {
- posLen = 1;
- }
- if (hasPayloads) {
- posLen++;
- }
- LowFreqDocsEnum docsEnum;
- if (reuse instanceof LowFreqDocsEnum) {
- docsEnum = (LowFreqDocsEnum) reuse;
- if (!docsEnum.canReuse(liveDocs, posLen)) {
- docsEnum = new LowFreqDocsEnum(liveDocs, posLen);
- }
- } else {
- docsEnum = new LowFreqDocsEnum(liveDocs, posLen);
- }
-
- return docsEnum.reset(postings);
- } else {
- LowFreqDocsEnumNoPos docsEnum;
- if (reuse instanceof LowFreqDocsEnumNoPos) {
- docsEnum = (LowFreqDocsEnumNoPos) reuse;
- if (!docsEnum.canReuse(liveDocs)) {
- docsEnum = new LowFreqDocsEnumNoPos(liveDocs);
- }
- } else {
- docsEnum = new LowFreqDocsEnumNoPos(liveDocs);
- }
-
- return docsEnum.reset(postings);
- }
- } else {
- LowFreqDocsEnumNoTF docsEnum;
- if (reuse instanceof LowFreqDocsEnumNoTF) {
- docsEnum = (LowFreqDocsEnumNoTF) reuse;
- if (!docsEnum.canReuse(liveDocs)) {
- docsEnum = new LowFreqDocsEnumNoTF(liveDocs);
- }
- } else {
- docsEnum = new LowFreqDocsEnumNoTF(liveDocs);
- }
-
- return docsEnum.reset(postings);
- }
- } else {
- final HighFreqTerm term = (HighFreqTerm) terms[termOrd];
-
- HighFreqDocsEnum docsEnum;
- if (reuse instanceof HighFreqDocsEnum) {
- docsEnum = (HighFreqDocsEnum) reuse;
- if (!docsEnum.canReuse(liveDocs)) {
- docsEnum = new HighFreqDocsEnum(liveDocs);
- }
- } else {
- docsEnum = new HighFreqDocsEnum(liveDocs);
- }
-
- //System.out.println(" DE for term=" + new BytesRef(terms[termOrd].term).utf8ToString() + ": " + term.docIDs.length + " docs");
- return docsEnum.reset(term.docIDs, term.freqs);
- }
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
- if (!hasPos) {
- return null;
- }
-
- // TODO: implement reuse, something like Pulsing:
- // it's hairy!
-
- if (terms[termOrd] instanceof LowFreqTerm) {
- final LowFreqTerm term = ((LowFreqTerm) terms[termOrd]);
- final int[] postings = term.postings;
- final byte[] payloads = term.payloads;
- return new LowFreqDocsAndPositionsEnum(liveDocs, hasOffsets, hasPayloads).reset(postings, payloads);
- } else {
- final HighFreqTerm term = (HighFreqTerm) terms[termOrd];
- return new HighFreqDocsAndPositionsEnum(liveDocs, hasOffsets).reset(term.docIDs, term.freqs, term.positions, term.payloads);
- }
- }
- }
-
- private final class DirectIntersectTermsEnum extends TermsEnum {
- private final RunAutomaton runAutomaton;
- private final CompiledAutomaton compiledAutomaton;
- private int termOrd;
- private final BytesRef scratch = new BytesRef();
-
- private final class State {
- int changeOrd;
- int state;
- Transition[] transitions;
- int transitionUpto;
- int transitionMax;
- int transitionMin;
- }
-
- private State[] states;
- private int stateUpto;
-
- public DirectIntersectTermsEnum(CompiledAutomaton compiled, BytesRef startTerm) {
- runAutomaton = compiled.runAutomaton;
- compiledAutomaton = compiled;
- termOrd = -1;
- states = new State[1];
- states[0] = new State();
- states[0].changeOrd = terms.length;
- states[0].state = runAutomaton.getInitialState();
- states[0].transitions = compiledAutomaton.sortedTransitions[states[0].state];
- states[0].transitionUpto = -1;
- states[0].transitionMax = -1;
-
- //System.out.println("IE.init startTerm=" + startTerm);
-
- if (startTerm != null) {
- int skipUpto = 0;
- if (startTerm.length == 0) {
- if (terms.length > 0 && termOffsets[1] == 0) {
- termOrd = 0;
- }
- } else {
- termOrd++;
-
- nextLabel:
- for(int i=0;i<startTerm.length;i++) {
- final int label = startTerm.bytes[startTerm.offset+i] & 0xFF;
-
- while (label > states[i].transitionMax) {
- states[i].transitionUpto++;
- Debug.Assert( states[i].transitionUpto < states[i].transitions.length;
- states[i].transitionMin = states[i].transitions[states[i].transitionUpto].getMin();
- states[i].transitionMax = states[i].transitions[states[i].transitionUpto].getMax();
- Debug.Assert( states[i].transitionMin >= 0;
- Debug.Assert( states[i].transitionMin <= 255;
- Debug.Assert( states[i].transitionMax >= 0;
- Debug.Assert( states[i].transitionMax <= 255;
- }
-
- // Skip forwards until we find a term matching
- // the label at this position:
- while (termOrd < terms.length) {
- final int skipOffset = skipOffsets[termOrd];
- final int numSkips = skipOffsets[termOrd+1] - skipOffset;
- final int termOffset = termOffsets[termOrd];
- final int termLength = termOffsets[1+termOrd] - termOffset;
-
- // if (DEBUG) {
- // System.out.println(" check termOrd=" + termOrd + " term=" + new BytesRef(termBytes, termOffset, termLength).utf8ToString() + " skips=" + Arrays.toString(skips) + " i=" + i);
- // }
-
- if (termOrd == states[stateUpto].changeOrd) {
- // if (DEBUG) {
- // System.out.println(" end push return");
- // }
- stateUpto--;
- termOrd--;
- return;
- }
-
- if (termLength == i) {
- termOrd++;
- skipUpto = 0;
- // if (DEBUG) {
- // System.out.println(" term too short; next term");
- // }
- } else if (label < (termBytes[termOffset+i] & 0xFF)) {
- termOrd--;
- // if (DEBUG) {
- // System.out.println(" no match; already beyond; return termOrd=" + termOrd);
- // }
- stateUpto -= skipUpto;
- Debug.Assert( stateUpto >= 0;
- return;
- } else if (label == (termBytes[termOffset+i] & 0xFF)) {
- // if (DEBUG) {
- // System.out.println(" label[" + i + "] matches");
- // }
- if (skipUpto < numSkips) {
- grow();
-
- final int nextState = runAutomaton.step(states[stateUpto].state, label);
-
- // Automaton is required to accept startTerm:
- Debug.Assert( nextState != -1;
-
- stateUpto++;
- states[stateUpto].changeOrd = skips[skipOffset + skipUpto++];
- states[stateUpto].state = nextState;
- states[stateUpto].transitions = compiledAutomaton.sortedTransitions[nextState];
- states[stateUpto].transitionUpto = -1;
- states[stateUpto].transitionMax = -1;
- //System.out.println(" push " + states[stateUpto].transitions.length + " trans");
-
- // if (DEBUG) {
- // System.out.println(" push skip; changeOrd=" + states[stateUpto].changeOrd);
- // }
-
- // Match next label at this same term:
- continue nextLabel;
- } else {
- // if (DEBUG) {
- // System.out.println(" linear scan");
- // }
- // Index exhausted: just scan now (the
- // number of scans required will be less
- // than the minSkipCount):
- final int startTermOrd = termOrd;
- while (termOrd < terms.length && compare(termOrd, startTerm) <= 0) {
- Debug.Assert( termOrd == startTermOrd || skipOffsets[termOrd] == skipOffsets[termOrd+1];
- termOrd++;
- }
- Debug.Assert( termOrd - startTermOrd < minSkipCount;
- termOrd--;
- stateUpto -= skipUpto;
- // if (DEBUG) {
- // System.out.println(" end termOrd=" + termOrd);
- // }
- return;
- }
- } else {
- if (skipUpto < numSkips) {
- termOrd = skips[skipOffset + skipUpto];
- // if (DEBUG) {
- // System.out.println(" no match; skip to termOrd=" + termOrd);
- // }
- } else {
- // if (DEBUG) {
- // System.out.println(" no match; next term");
- // }
- termOrd++;
- }
- skipUpto = 0;
- }
- }
-
- // startTerm is >= last term so enum will not
- // return any terms:
- termOrd--;
- // if (DEBUG) {
- // System.out.println(" beyond end; no terms will match");
- // }
- return;
- }
- }
-
- final int termOffset = termOffsets[termOrd];
- final int termLen = termOffsets[1+termOrd] - termOffset;
-
- if (termOrd >= 0 && !startTerm.equals(new BytesRef(termBytes, termOffset, termLen))) {
- stateUpto -= skipUpto;
- termOrd--;
- }
- // if (DEBUG) {
- // System.out.println(" loop end; return termOrd=" + termOrd + " stateUpto=" + stateUpto);
- // }
- }
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- private void grow() {
- if (states.length == 1+stateUpto) {
- final State[] newStates = new State[states.length+1];
- System.arraycopy(states, 0, newStates, 0, states.length);
- newStates[states.length] = new State();
- states = newStates;
- }
- }
-
- @Override
- public BytesRef next() {
- // if (DEBUG) {
- // System.out.println("\nIE.next");
- // }
-
- termOrd++;
- int skipUpto = 0;
-
- if (termOrd == 0 && termOffsets[1] == 0) {
- // Special-case empty string:
- Debug.Assert( stateUpto == 0;
- // if (DEBUG) {
- // System.out.println(" visit empty string");
- // }
- if (runAutomaton.isAccept(states[0].state)) {
- scratch.bytes = termBytes;
- scratch.offset = 0;
- scratch.length = 0;
- return scratch;
- }
- termOrd++;
- }
-
- nextTerm:
-
- while (true) {
- // if (DEBUG) {
- // System.out.println(" cycle termOrd=" + termOrd + " stateUpto=" + stateUpto + " skipUpto=" + skipUpto);
- // }
- if (termOrd == terms.length) {
- // if (DEBUG) {
- // System.out.println(" return END");
- // }
- return null;
- }
-
- final State state = states[stateUpto];
- if (termOrd == state.changeOrd) {
- // Pop:
- // if (DEBUG) {
- // System.out.println(" pop stateUpto=" + stateUpto);
- // }
- stateUpto--;
- /*
- if (DEBUG) {
- try {
- //System.out.println(" prefix pop " + new BytesRef(terms[termOrd].term, 0, Math.min(stateUpto, terms[termOrd].term.length)).utf8ToString());
- System.out.println(" prefix pop " + new BytesRef(terms[termOrd].term, 0, Math.min(stateUpto, terms[termOrd].term.length)));
- } catch (ArrayIndexOutOfBoundsException aioobe) {
- System.out.println(" prefix pop " + new BytesRef(terms[termOrd].term, 0, Math.min(stateUpto, terms[termOrd].term.length)));
- }
- }
- */
-
- continue;
- }
-
- final int termOffset = termOffsets[termOrd];
- final int termLength = termOffsets[termOrd+1] - termOffset;
- final int skipOffset = skipOffsets[termOrd];
- final int numSkips = skipOffsets[termOrd+1] - skipOffset;
-
- // if (DEBUG) {
- // System.out.println(" term=" + new BytesRef(termBytes, termOffset, termLength).utf8ToString() + " skips=" + Arrays.toString(skips));
- // }
-
- Debug.Assert( termOrd < state.changeOrd;
-
- Debug.Assert( stateUpto <= termLength: "term.length=" + termLength + "; stateUpto=" + stateUpto;
- final int label = termBytes[termOffset+stateUpto] & 0xFF;
-
- while (label > state.transitionMax) {
- //System.out.println(" label=" + label + " vs max=" + state.transitionMax + " transUpto=" + state.transitionUpto + " vs " + state.transitions.length);
- state.transitionUpto++;
- if (state.transitionUpto == state.transitions.length) {
- // We've exhausted transitions leaving this
- // state; force pop+next/skip now:
- //System.out.println("forcepop: stateUpto=" + stateUpto);
- if (stateUpto == 0) {
- termOrd = terms.length;
- return null;
- } else {
- Debug.Assert( state.changeOrd > termOrd;
- // if (DEBUG) {
- // System.out.println(" jumpend " + (state.changeOrd - termOrd));
- // }
- //System.out.println(" jump to termOrd=" + states[stateUpto].changeOrd + " vs " + termOrd);
- termOrd = states[stateUpto].changeOrd;
- skipUpto = 0;
- stateUpto--;
- }
- continue nextTerm;
- }
- Debug.Assert( state.transitionUpto < state.transitions.length: " state.transitionUpto=" + state.transitionUpto + " vs " + state.transitions.length;
- state.transitionMin = state.transitions[state.transitionUpto].getMin();
- state.transitionMax = state.transitions[state.transitionUpto].getMax();
- Debug.Assert( state.transitionMin >= 0;
- Debug.Assert( state.transitionMin <= 255;
- Debug.Assert( state.transitionMax >= 0;
- Debug.Assert( state.transitionMax <= 255;
- }
-
- /*
- if (DEBUG) {
- System.out.println(" check ord=" + termOrd + " term[" + stateUpto + "]=" + (char) label + "(" + label + ") term=" + new BytesRef(terms[termOrd].term).utf8ToString() + " trans " +
- (char) state.transitionMin + "(" + state.transitionMin + ")" + "-" + (char) state.transitionMax + "(" + state.transitionMax + ") nextChange=+" + (state.changeOrd - termOrd) + " skips=" + (skips == null ? "null" : Arrays.toString(skips)));
- System.out.println(" check ord=" + termOrd + " term[" + stateUpto + "]=" + Integer.toHexString(label) + "(" + label + ") term=" + new BytesRef(termBytes, termOffset, termLength) + " trans " +
- Integer.toHexString(state.transitionMin) + "(" + state.transitionMin + ")" + "-" + Integer.toHexString(state.transitionMax) + "(" + state.transitionMax + ") nextChange=+" + (state.changeOrd - termOrd) + " skips=" + (skips == null ? "null" : Arrays.toString(skips)));
- }
- */
-
- final int targetLabel = state.transitionMin;
-
- if ((termBytes[termOffset+stateUpto] & 0xFF) < targetLabel) {
- // if (DEBUG) {
- // System.out.println(" do bin search");
- // }
- //int startTermOrd = termOrd;
- int low = termOrd+1;
- int high = state.changeOrd-1;
- while (true) {
- if (low > high) {
- // Label not found
- termOrd = low;
- // if (DEBUG) {
- // System.out.println(" advanced by " + (termOrd - startTermOrd));
- // }
- //System.out.println(" jump " + (termOrd - startTermOrd));
- skipUpto = 0;
- continue nextTerm;
- }
- int mid = (low + high) >>> 1;
- int cmp = (termBytes[termOffsets[mid] + stateUpto] & 0xFF) - targetLabel;
- // if (DEBUG) {
- // System.out.println(" bin: check label=" + (char) (termBytes[termOffsets[low] + stateUpto] & 0xFF) + " ord=" + mid);
- // }
- if (cmp < 0) {
- low = mid+1;
- } else if (cmp > 0) {
- high = mid - 1;
- } else {
- // Label found; walk backwards to first
- // occurrence:
- while (mid > termOrd && (termBytes[termOffsets[mid-1] + stateUpto] & 0xFF) == targetLabel) {
- mid--;
- }
- termOrd = mid;
- // if (DEBUG) {
- // System.out.println(" advanced by " + (termOrd - startTermOrd));
- // }
- //System.out.println(" jump " + (termOrd - startTermOrd));
- skipUpto = 0;
- continue nextTerm;
- }
- }
- }
-
- int nextState = runAutomaton.step(states[stateUpto].state, label);
-
- if (nextState == -1) {
- // Skip
- // if (DEBUG) {
- // System.out.println(" automaton doesn't accept; skip");
- // }
- if (skipUpto < numSkips) {
- // if (DEBUG) {
- // System.out.println(" jump " + (skips[skipOffset+skipUpto]-1 - termOrd));
- // }
- termOrd = skips[skipOffset+skipUpto];
- } else {
- termOrd++;
- }
- skipUpto = 0;
- } else if (skipUpto < numSkips) {
- // Push:
- // if (DEBUG) {
- // System.out.println(" push");
- // }
- /*
- if (DEBUG) {
- try {
- //System.out.println(" prefix push " + new BytesRef(term, 0, stateUpto+1).utf8ToString());
- System.out.println(" prefix push " + new BytesRef(term, 0, stateUpto+1));
- } catch (ArrayIndexOutOfBoundsException aioobe) {
- System.out.println(" prefix push " + new BytesRef(term, 0, stateUpto+1));
- }
- }
- */
-
- grow();
- stateUpto++;
- states[stateUpto].state = nextState;
- states[stateUpto].changeOrd = skips[skipOffset + skipUpto++];
- states[stateUpto].transitions = compiledAutomaton.sortedTransitions[nextState];
- states[stateUpto].transitionUpto = -1;
- states[stateUpto].transitionMax = -1;
-
- if (stateUpto == termLength) {
- // if (DEBUG) {
- // System.out.println(" term ends after push");
- // }
- if (runAutomaton.isAccept(nextState)) {
- // if (DEBUG) {
- // System.out.println(" automaton accepts: return");
- // }
- scratch.bytes = termBytes;
- scratch.offset = termOffsets[termOrd];
- scratch.length = termOffsets[1+termOrd] - scratch.offset;
- // if (DEBUG) {
- // System.out.println(" ret " + scratch.utf8ToString());
- // }
- return scratch;
- } else {
- // if (DEBUG) {
- // System.out.println(" automaton rejects: nextTerm");
- // }
- termOrd++;
- skipUpto = 0;
- }
- }
- } else {
- // Run the non-indexed tail of this term:
-
- // TODO: add Debug.Assert( that we don't inc too many times
-
- if (compiledAutomaton.commonSuffixRef != null) {
- //System.out.println("suffix " + compiledAutomaton.commonSuffixRef.utf8ToString());
- Debug.Assert( compiledAutomaton.commonSuffixRef.offset == 0;
- if (termLength < compiledAutomaton.commonSuffixRef.length) {
- termOrd++;
- skipUpto = 0;
- continue nextTerm;
- }
- int offset = termOffset + termLength - compiledAutomaton.commonSuffixRef.length;
- for(int suffix=0;suffix<compiledAutomaton.commonSuffixRef.length;suffix++) {
- if (termBytes[offset + suffix] != compiledAutomaton.commonSuffixRef.bytes[suffix]) {
- termOrd++;
- skipUpto = 0;
- continue nextTerm;
- }
- }
- }
-
- int upto = stateUpto+1;
- while (upto < termLength) {
- nextState = runAutomaton.step(nextState, termBytes[termOffset+upto] & 0xFF);
- if (nextState == -1) {
- termOrd++;
- skipUpto = 0;
- // if (DEBUG) {
- // System.out.println(" nomatch tail; next term");
- // }
- continue nextTerm;
- }
- upto++;
- }
-
- if (runAutomaton.isAccept(nextState)) {
- scratch.bytes = termBytes;
- scratch.offset = termOffsets[termOrd];
- scratch.length = termOffsets[1+termOrd] - scratch.offset;
- // if (DEBUG) {
- // System.out.println(" match tail; return " + scratch.utf8ToString());
- // System.out.println(" ret2 " + scratch.utf8ToString());
- // }
- return scratch;
- } else {
- termOrd++;
- skipUpto = 0;
- // if (DEBUG) {
- // System.out.println(" nomatch tail; next term");
- // }
- }
- }
- }
- }
-
- @Override
- public TermState termState() {
- OrdTermState state = new OrdTermState();
- state.ord = termOrd;
- return state;
- }
-
- @Override
- public BytesRef term() {
- return scratch;
- }
-
- @Override
- public long ord() {
- return termOrd;
- }
-
- @Override
- public int docFreq() {
- if (terms[termOrd] instanceof LowFreqTerm) {
- return ((LowFreqTerm) terms[termOrd]).docFreq;
- } else {
- return ((HighFreqTerm) terms[termOrd]).docIDs.length;
- }
- }
-
- @Override
- public long totalTermFreq() {
- if (terms[termOrd] instanceof LowFreqTerm) {
- return ((LowFreqTerm) terms[termOrd]).totalTermFreq;
- } else {
- return ((HighFreqTerm) terms[termOrd]).totalTermFreq;
- }
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
- // TODO: implement reuse, something like Pulsing:
- // it's hairy!
-
- if (terms[termOrd] instanceof LowFreqTerm) {
- final int[] postings = ((LowFreqTerm) terms[termOrd]).postings;
- if (hasFreq) {
- if (hasPos) {
- int posLen;
- if (hasOffsets) {
- posLen = 3;
- } else {
- posLen = 1;
- }
- if (hasPayloads) {
- posLen++;
- }
- return new LowFreqDocsEnum(liveDocs, posLen).reset(postings);
- } else {
- return new LowFreqDocsEnumNoPos(liveDocs).reset(postings);
- }
- } else {
- return new LowFreqDocsEnumNoTF(liveDocs).reset(postings);
- }
- } else {
- final HighFreqTerm term = (HighFreqTerm) terms[termOrd];
- // System.out.println("DE for term=" + new BytesRef(terms[termOrd].term).utf8ToString() + ": " + term.docIDs.length + " docs");
- return new HighFreqDocsEnum(liveDocs).reset(term.docIDs, term.freqs);
- }
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
- if (!hasPos) {
- return null;
- }
-
- // TODO: implement reuse, something like Pulsing:
- // it's hairy!
-
- if (terms[termOrd] instanceof LowFreqTerm) {
- final LowFreqTerm term = ((LowFreqTerm) terms[termOrd]);
- final int[] postings = term.postings;
- final byte[] payloads = term.payloads;
- return new LowFreqDocsAndPositionsEnum(liveDocs, hasOffsets, hasPayloads).reset(postings, payloads);
- } else {
- final HighFreqTerm term = (HighFreqTerm) terms[termOrd];
- return new HighFreqDocsAndPositionsEnum(liveDocs, hasOffsets).reset(term.docIDs, term.freqs, term.positions, term.payloads);
- }
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef term) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void seekExact(long ord) {
- throw new UnsupportedOperationException();
- }
- }
- }
-
- // Docs only:
- private final static class LowFreqDocsEnumNoTF extends DocsEnum {
- private int[] postings;
- private final Bits liveDocs;
- private int upto;
-
- public LowFreqDocsEnumNoTF(Bits liveDocs) {
- this.liveDocs = liveDocs;
- }
-
- public bool canReuse(Bits liveDocs) {
- return liveDocs == this.liveDocs;
- }
-
- public DocsEnum reset(int[] postings) {
- this.postings = postings;
- upto = -1;
- return this;
- }
-
- // TODO: can do this w/o setting members?
-
- @Override
- public int nextDoc() {
- upto++;
- if (liveDocs == null) {
- if (upto < postings.length) {
- return postings[upto];
- }
- } else {
- while (upto < postings.length) {
- if (liveDocs.get(postings[upto])) {
- return postings[upto];
- }
- upto++;
- }
- }
- return NO_MORE_DOCS;
- }
-
- @Override
- public int docID() {
- if (upto < 0) {
- return -1;
- } else if (upto < postings.length) {
- return postings[upto];
- } else {
- return NO_MORE_DOCS;
- }
- }
-
- @Override
- public int freq() {
- return 1;
- }
-
- @Override
- public int advance(int target) {
- // Linear scan, but this is low-freq term so it won't
- // be costly:
- return slowAdvance(target);
- }
-
- @Override
- public long cost() {
- return postings.length;
- }
- }
-
- // Docs + freqs:
- private final static class LowFreqDocsEnumNoPos extends DocsEnum {
- private int[] postings;
- private final Bits liveDocs;
- private int upto;
-
- public LowFreqDocsEnumNoPos(Bits liveDocs) {
- this.liveDocs = liveDocs;
- }
-
- public bool canReuse(Bits liveDocs) {
- return liveDocs == this.liveDocs;
- }
-
- public DocsEnum reset(int[] postings) {
- this.postings = postings;
- upto = -2;
- return this;
- }
-
- // TODO: can do this w/o setting members?
- @Override
- public int nextDoc() {
- upto += 2;
- if (liveDocs == null) {
- if (upto < postings.length) {
- return postings[upto];
- }
- } else {
- while (upto < postings.length) {
- if (liveDocs.get(postings[upto])) {
- return postings[upto];
- }
- upto += 2;
- }
- }
- return NO_MORE_DOCS;
- }
-
- @Override
- public int docID() {
- if (upto < 0) {
- return -1;
- } else if (upto < postings.length) {
- return postings[upto];
- } else {
- return NO_MORE_DOCS;
- }
- }
-
- @Override
- public int freq() {
- return postings[upto+1];
- }
-
- @Override
- public int advance(int target) {
- // Linear scan, but this is low-freq term so it won't
- // be costly:
- return slowAdvance(target);
- }
-
- @Override
- public long cost() {
- return postings.length / 2;
- }
- }
-
- // Docs + freqs + positions/offets:
- private final static class LowFreqDocsEnum extends DocsEnum {
- private int[] postings;
- private final Bits liveDocs;
- private final int posMult;
- private int upto;
- private int freq;
-
- public LowFreqDocsEnum(Bits liveDocs, int posMult) {
- this.liveDocs = liveDocs;
- this.posMult = posMult;
- // if (DEBUG) {
- // System.out.println("LowFreqDE: posMult=" + posMult);
- // }
- }
-
- public bool canReuse(Bits liveDocs, int posMult) {
- return liveDocs == this.liveDocs && posMult == this.posMult;
- }
-
- public DocsEnum reset(int[] postings) {
- this.postings = postings;
- upto = -2;
- freq = 0;
- return this;
- }
-
- // TODO: can do this w/o setting members?
- @Override
- public int nextDoc() {
- upto += 2 + freq*posMult;
- // if (DEBUG) {
- // System.out.println(" nextDoc freq=" + freq + " upto=" + upto + " vs " + postings.length);
- // }
- if (liveDocs == null) {
- if (upto < postings.length) {
- freq = postings[upto+1];
- Debug.Assert( freq > 0;
- return postings[upto];
- }
- } else {
- while (upto < postings.length) {
- freq = postings[upto+1];
- Debug.Assert( freq > 0;
- if (liveDocs.get(postings[upto])) {
- return postings[upto];
- }
- upto += 2 + freq*posMult;
- }
- }
- return NO_MORE_DOCS;
- }
-
- @Override
- public int docID() {
- // TODO: store docID member?
- if (upto < 0) {
- return -1;
- } else if (upto < postings.length) {
- return postings[upto];
- } else {
- return NO_MORE_DOCS;
- }
- }
-
- @Override
- public int freq() {
- // TODO: can I do postings[upto+1]?
- return freq;
- }
-
- @Override
- public int advance(int target) {
- // Linear scan, but this is low-freq term so it won't
- // be costly:
- return slowAdvance(target);
- }
-
- @Override
- public long cost() {
- // TODO: could do a better estimate
- return postings.length / 2;
- }
- }
-
- private final static class LowFreqDocsAndPositionsEnum extends DocsAndPositionsEnum {
- private int[] postings;
- private final Bits liveDocs;
- private final int posMult;
- private final bool hasOffsets;
- private final bool hasPayloads;
- private final BytesRef payload = new BytesRef();
- private int upto;
- private int docID;
- private int freq;
- private int skipPositions;
- private int startOffset;
- private int endOffset;
- private int lastPayloadOffset;
- private int payloadOffset;
- private int payloadLength;
- private byte[] payloadBytes;
-
- public LowFreqDocsAndPositionsEnum(Bits liveDocs, bool hasOffsets, bool hasPayloads) {
- this.liveDocs = liveDocs;
- this.hasOffsets = hasOffsets;
- this.hasPayloads = hasPayloads;
- if (hasOffsets) {
- if (hasPayloads) {
- posMult = 4;
- } else {
- posMult = 3;
- }
- } else if (hasPayloads) {
- posMult = 2;
- } else {
- posMult = 1;
- }
- }
-
- public DocsAndPositionsEnum reset(int[] postings, byte[] payloadBytes) {
- this.postings = postings;
- upto = 0;
- skipPositions = 0;
- startOffset = -1;
- endOffset = -1;
- docID = -1;
- payloadLength = 0;
- this.payloadBytes = payloadBytes;
- return this;
- }
-
- @Override
- public int nextDoc() {
- if (hasPayloads) {
- for(int i=0;i<skipPositions;i++) {
- upto++;
- if (hasOffsets) {
- upto += 2;
- }
- payloadOffset += postings[upto++];
- }
- } else {
- upto += posMult * skipPositions;
- }
-
- if (liveDocs == null) {
- if (upto < postings.length) {
- docID = postings[upto++];
- freq = postings[upto++];
- skipPositions = freq;
- return docID;
- }
- } else {
- while(upto < postings.length) {
- docID = postings[upto++];
- freq = postings[upto++];
- if (liveDocs.get(docID)) {
- skipPositions = freq;
- return docID;
- }
- if (hasPayloads) {
- for(int i=0;i<freq;i++) {
- upto++;
- if (hasOffsets) {
- upto += 2;
- }
- payloadOffset += postings[upto++];
- }
- } else {
- upto += posMult * freq;
- }
- }
- }
-
- return docID = NO_MORE_DOCS;
- }
-
- @Override
- public int docID() {
- return docID;
- }
-
- @Override
- public int freq() {
- return freq;
- }
-
- @Override
- public int nextPosition() {
- Debug.Assert( skipPositions > 0;
- skipPositions--;
- final int pos = postings[upto++];
- if (hasOffsets) {
- startOffset = postings[upto++];
- endOffset = postings[upto++];
- }
- if (hasPayloads) {
- payloadLength = postings[upto++];
- lastPayloadOffset = payloadOffset;
- payloadOffset += payloadLength;
- }
- return pos;
- }
-
- @Override
- public int startOffset() {
- return startOffset;
- }
-
- @Override
- public int endOffset() {
- return endOffset;
- }
-
- @Override
- public int advance(int target) {
- return slowAdvance(target);
- }
-
- @Override
- public BytesRef getPayload() {
- if (payloadLength > 0) {
- payload.bytes = payloadBytes;
- payload.offset = lastPayloadOffset;
- payload.length = payloadLength;
- return payload;
- } else {
- return null;
- }
- }
-
- @Override
- public long cost() {
- // TODO: could do a better estimate
- return postings.length / 2;
- }
- }
-
- // Docs + freqs:
- private final static class HighFreqDocsEnum extends DocsEnum {
- private int[] docIDs;
- private int[] freqs;
- private final Bits liveDocs;
- private int upto;
- private int docID = -1;
-
- public HighFreqDocsEnum(Bits liveDocs) {
- this.liveDocs = liveDocs;
- }
-
- public bool canReuse(Bits liveDocs) {
- return liveDocs == this.liveDocs;
- }
-
- public int[] getDocIDs() {
- return docIDs;
- }
-
- public int[] getFreqs() {
- return freqs;
- }
-
- public DocsEnum reset(int[] docIDs, int[] freqs) {
- this.docIDs = docIDs;
- this.freqs = freqs;
- docID = upto = -1;
- return this;
- }
-
- @Override
- public int nextDoc() {
- upto++;
- if (liveDocs == null) {
- try {
- return docID = docIDs[upto];
- } catch (ArrayIndexOutOfBoundsException e) {
- }
- } else {
- while (upto < docIDs.length) {
- if (liveDocs.get(docIDs[upto])) {
- return docID = docIDs[upto];
- }
- upto++;
- }
- }
- return docID = NO_MORE_DOCS;
- }
-
- @Override
- public int docID() {
- return docID;
- }
-
- @Override
- public int freq() {
- if (freqs == null) {
- return 1;
- } else {
- return freqs[upto];
- }
- }
-
- @Override
- public int advance(int target) {
- /*
- upto++;
- if (upto == docIDs.length) {
- return docID = NO_MORE_DOCS;
- }
- final int index = Arrays.binarySearch(docIDs, upto, docIDs.length, target);
- if (index < 0) {
- upto = -index - 1;
- } else {
- upto = index;
- }
- if (liveDocs != null) {
- while (upto < docIDs.length) {
- if (liveDocs.get(docIDs[upto])) {
- break;
- }
- upto++;
- }
- }
- if (upto == docIDs.length) {
- return NO_MORE_DOCS;
- } else {
- return docID = docIDs[upto];
- }
- */
-
- //System.out.println(" advance target=" + target + " cur=" + docID() + " upto=" + upto + " of " + docIDs.length);
- // if (DEBUG) {
- // System.out.println("advance target=" + target + " len=" + docIDs.length);
- // }
- upto++;
- if (upto == docIDs.length) {
- return docID = NO_MORE_DOCS;
- }
-
- // First "grow" outwards, since most advances are to
- // nearby docs:
- int inc = 10;
- int nextUpto = upto+10;
- int low;
- int high;
- while (true) {
- //System.out.println(" grow nextUpto=" + nextUpto + " inc=" + inc);
- if (nextUpto >= docIDs.length) {
- low = nextUpto-inc;
- high = docIDs.length-1;
- break;
- }
- //System.out.println(" docID=" + docIDs[nextUpto]);
-
- if (target <= docIDs[nextUpto]) {
- low = nextUpto-inc;
- high = nextUpto;
- break;
- }
- inc *= 2;
- nextUpto += inc;
- }
-
- // Now do normal binary search
- //System.out.println(" after fwd: low=" + low + " high=" + high);
-
- while (true) {
-
- if (low > high) {
- // Not exactly found
- //System.out.println(" break: no match");
- upto = low;
- break;
- }
-
- int mid = (low + high) >>> 1;
- int cmp = docIDs[mid] - target;
- //System.out.println(" bsearch low=" + low + " high=" + high+ ": docIDs[" + mid + "]=" + docIDs[mid]);
-
- if (cmp < 0) {
- low = mid + 1;
- } else if (cmp > 0) {
- high = mid - 1;
- } else {
- // Found target
- upto = mid;
- //System.out.println(" break: match");
- break;
- }
- }
-
- //System.out.println(" end upto=" + upto + " docID=" + (upto >= docIDs.length ? NO_MORE_DOCS : docIDs[upto]));
-
- if (liveDocs != null) {
- while (upto < docIDs.length) {
- if (liveDocs.get(docIDs[upto])) {
- break;
- }
- upto++;
- }
- }
- if (upto == docIDs.length) {
- //System.out.println(" return END");
- return docID = NO_MORE_DOCS;
- } else {
- //System.out.println(" return docID=" + docIDs[upto] + " upto=" + upto);
- return docID = docIDs[upto];
- }
- }
-
- @Override
- public long cost() {
- return docIDs.length;
- }
- }
-
- // TODO: specialize offsets and not
- private final static class HighFreqDocsAndPositionsEnum extends DocsAndPositionsEnum {
- private int[] docIDs;
- private int[] freqs;
- private int[][] positions;
- private byte[][][] payloads;
- private final Bits liveDocs;
- private final bool hasOffsets;
- private final int posJump;
- private int upto;
- private int docID = -1;
- private int posUpto;
- private int[] curPositions;
-
- public HighFreqDocsAndPositionsEnum(Bits liveDocs, bool hasOffsets) {
- this.liveDocs = liveDocs;
- this.hasOffsets = hasOffsets;
- posJump = hasOffsets ? 3 : 1;
- }
-
- public int[] getDocIDs() {
- return docIDs;
- }
-
- public int[][] getPositions() {
- return positions;
- }
-
- public int getPosJump() {
- return posJump;
- }
-
- public Bits getLiveDocs() {
- return liveDocs;
- }
-
- public DocsAndPositionsEnum reset(int[] docIDs, int[] freqs, int[][] positions, byte[][][] payloads) {
- this.docIDs = docIDs;
- this.freqs = freqs;
- this.positions = positions;
- this.payloads = payloads;
- upto = -1;
- return this;
- }
-
- @Override
- public int nextDoc() {
- upto++;
- if (liveDocs == null) {
- if (upto < docIDs.length) {
- posUpto = -posJump;
- curPositions = positions[upto];
- return docID = docIDs[upto];
- }
- } else {
- while (upto < docIDs.length) {
- if (liveDocs.get(docIDs[upto])) {
- posUpto = -posJump;
- curPositions = positions[upto];
- return docID = docIDs[upto];
- }
- upto++;
- }
- }
-
- return docID = NO_MORE_DOCS;
- }
-
- @Override
- public int freq() {
- return freqs[upto];
- }
-
- @Override
- public int docID() {
- return docID;
- }
-
- @Override
- public int nextPosition() {
- posUpto += posJump;
- return curPositions[posUpto];
- }
-
- @Override
- public int startOffset() {
- if (hasOffsets) {
- return curPositions[posUpto+1];
- } else {
- return -1;
- }
- }
-
- @Override
- public int endOffset() {
- if (hasOffsets) {
- return curPositions[posUpto+2];
- } else {
- return -1;
- }
- }
-
- @Override
- public int advance(int target) {
-
- /*
- upto++;
- if (upto == docIDs.length) {
- return NO_MORE_DOCS;
- }
- final int index = Arrays.binarySearch(docIDs, upto, docIDs.length, target);
- if (index < 0) {
- upto = -index - 1;
- } else {
- upto = index;
- }
- if (liveDocs != null) {
- while (upto < docIDs.length) {
- if (liveDocs.get(docIDs[upto])) {
- break;
- }
- upto++;
- }
- }
- posUpto = hasOffsets ? -3 : -1;
- if (upto == docIDs.length) {
- return NO_MORE_DOCS;
- } else {
- return docID();
- }
- */
-
- //System.out.println(" advance target=" + target + " cur=" + docID() + " upto=" + upto + " of " + docIDs.length);
- // if (DEBUG) {
- // System.out.println("advance target=" + target + " len=" + docIDs.length);
- // }
- upto++;
- if (upto == docIDs.length) {
- return docID = NO_MORE_DOCS;
- }
-
- // First "grow" outwards, since most advances are to
- // nearby docs:
- int inc = 10;
- int nextUpto = upto+10;
- int low;
- int high;
- while (true) {
- //System.out.println(" grow nextUpto=" + nextUpto + " inc=" + inc);
- if (nextUpto >= docIDs.length) {
- low = nextUpto-inc;
- high = docIDs.length-1;
- break;
- }
- //System.out.println(" docID=" + docIDs[nextUpto]);
-
- if (target <= docIDs[nextUpto]) {
- low = nextUpto-inc;
- high = nextUpto;
- break;
- }
- inc *= 2;
- nextUpto += inc;
- }
-
- // Now do normal binary search
- //System.out.println(" after fwd: low=" + low + " high=" + high);
-
- while (true) {
-
- if (low > high) {
- // Not exactly found
- //System.out.println(" break: no match");
- upto = low;
- break;
- }
-
- int mid = (low + high) >>> 1;
- int cmp = docIDs[mid] - target;
- //System.out.println(" bsearch low=" + low + " high=" + high+ ": docIDs[" + mid + "]=" + docIDs[mid]);
-
- if (cmp < 0) {
- low = mid + 1;
- } else if (cmp > 0) {
- high = mid - 1;
- } else {
- // Found target
- upto = mid;
- //System.out.println(" break: match");
- break;
- }
- }
-
- //System.out.println(" end upto=" + upto + " docID=" + (upto >= docIDs.length ? NO_MORE_DOCS : docIDs[upto]));
-
- if (liveDocs != null) {
- while (upto < docIDs.length) {
- if (liveDocs.get(docIDs[upto])) {
- break;
- }
- upto++;
- }
- }
- if (upto == docIDs.length) {
- //System.out.println(" return END");
- return docID = NO_MORE_DOCS;
- } else {
- //System.out.println(" return docID=" + docIDs[upto] + " upto=" + upto);
- posUpto = -posJump;
- curPositions = positions[upto];
- return docID = docIDs[upto];
- }
- }
-
- private final BytesRef payload = new BytesRef();
-
- @Override
- public BytesRef getPayload() {
- if (payloads == null) {
- return null;
- } else {
- final byte[] payloadBytes = payloads[upto][posUpto/(hasOffsets ? 3:1)];
- if (payloadBytes == null) {
- return null;
- }
- payload.bytes = payloadBytes;
- payload.length = payloadBytes.length;
- payload.offset = 0;
- return payload;
- }
- }
-
- @Override
- public long cost() {
- return docIDs.length;
- }
- }
-}
+using System;
+using System.Diagnostics;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /// <summary>
+ /// Licensed to the Apache Software Foundation (ASF) under one or more
+ /// contributor license agreements. See the NOTICE file distributed with
+ /// this work for additional information regarding copyright ownership.
+ /// The ASF licenses this file to You under the Apache License, Version 2.0
+ /// (the "License"); you may not use this file except in compliance with
+ /// the License. You may obtain a copy of the License at
+ ///
+ /// http://www.apache.org/licenses/LICENSE-2.0
+ ///
+ /// Unless required by applicable law or agreed to in writing, software
+ /// distributed under the License is distributed on an "AS IS" BASIS,
+ /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ /// See the License for the specific language governing permissions and
+ /// limitations under the License.
+ /// </summary>
+
+
+ using Lucene41PostingsFormat = org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat; // javadocs
+ using DocsAndPositionsEnum = org.apache.lucene.index.DocsAndPositionsEnum;
+ using DocsEnum = org.apache.lucene.index.DocsEnum;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using Fields = org.apache.lucene.index.Fields;
+ using OrdTermState = org.apache.lucene.index.OrdTermState;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using TermState = org.apache.lucene.index.TermState;
+ using Terms = org.apache.lucene.index.Terms;
+ using TermsEnum = org.apache.lucene.index.TermsEnum;
+ using IOContext = org.apache.lucene.store.IOContext;
+ using RAMOutputStream = org.apache.lucene.store.RAMOutputStream;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using Bits = org.apache.lucene.util.Bits;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using RamUsageEstimator = org.apache.lucene.util.RamUsageEstimator;
+ using CompiledAutomaton = org.apache.lucene.util.automaton.CompiledAutomaton;
+ using RunAutomaton = org.apache.lucene.util.automaton.RunAutomaton;
+ using Transition = org.apache.lucene.util.automaton.Transition;
+
+ // TODO:
+ // - build depth-N prefix hash?
+ // - or: longer dense skip lists than just next byte?
+
+ /// <summary>
+ /// Wraps <seealso cref="Lucene41PostingsFormat"/> format for on-disk
+ /// storage, but then at read time loads and stores all
+ /// terms & postings directly in RAM as byte[], int[].
+ ///
+ /// <para><b><font color=red>WARNING</font></b>: This is
+ /// exceptionally RAM intensive: it makes no effort to
+ /// compress the postings data, storing terms as separate
+ /// byte[] and postings as separate int[], but as a result it
+ /// gives substantial increase in search performance.
+ ///
+ /// </para>
+ /// <para>This postings format supports <seealso cref="TermsEnum#ord"/>
+ /// and <seealso cref="TermsEnum#seekExact(long)"/>.
+ ///
+ /// </para>
+ /// <para>Because this holds all term bytes as a single
+ /// byte[], you cannot have more than 2.1GB worth of term
+ /// bytes in a single segment.
+ ///
+ /// @lucene.experimental
+ /// </para>
+ /// </summary>
+
+ public sealed class DirectPostingsFormat : PostingsFormat
+ {
+
+ private readonly int minSkipCount;
+ private readonly int lowFreqCutoff;
+
+ private const int DEFAULT_MIN_SKIP_COUNT = 8;
+ private const int DEFAULT_LOW_FREQ_CUTOFF = 32;
+
+ //private static final boolean DEBUG = true;
+
+ // TODO: allow passing/wrapping arbitrary postings format?
+
+ public DirectPostingsFormat() : this(DEFAULT_MIN_SKIP_COUNT, DEFAULT_LOW_FREQ_CUTOFF)
+ {
+ }
+
+ /// <summary>
+ /// minSkipCount is how many terms in a row must have the
+ /// same prefix before we put a skip pointer down. Terms
+ /// with docFreq <= lowFreqCutoff will use a single int[]
+ /// to hold all docs, freqs, position and offsets; terms
+ /// with higher docFreq will use separate arrays.
+ /// </summary>
+ public DirectPostingsFormat(int minSkipCount, int lowFreqCutoff) : base("Direct")
+ {
+ this.minSkipCount = minSkipCount;
+ this.lowFreqCutoff = lowFreqCutoff;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ return PostingsFormat.forName("Lucene41").fieldsConsumer(state);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override FieldsProducer fieldsProducer(SegmentReadState state)
+ {
+ FieldsProducer postings = PostingsFormat.forName("Lucene41").fieldsProducer(state);
+ if (state.context.context != IOContext.Context.MERGE)
+ {
+ FieldsProducer loadedPostings;
+ try
+ {
+ postings.checkIntegrity();
+ loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff);
+ }
+ finally
+ {
+ postings.close();
+ }
+ return loadedPostings;
+ }
+ else
+ {
+ // Don't load postings for merge:
+ return postings;
+ }
+ }
+
+ private sealed class DirectFields : FieldsProducer
+ {
+ internal readonly IDictionary<string, DirectField> fields = new SortedDictionary<string, DirectField>();
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public DirectFields(org.apache.lucene.index.SegmentReadState state, org.apache.lucene.index.Fields fields, int minSkipCount, int lowFreqCutoff) throws java.io.IOException
+ public DirectFields(SegmentReadState state, Fields fields, int minSkipCount, int lowFreqCutoff)
+ {
+ foreach (string field in fields)
+ {
+ this.fields[field] = new DirectField(state, field, fields.terms(field), minSkipCount, lowFreqCutoff);
+ }
+ }
+
+ public override IEnumerator<string> iterator()
+ {
+ return Collections.unmodifiableSet(fields.Keys).GetEnumerator();
+ }
+
+ public override Terms terms(string field)
+ {
+ return fields[field];
+ }
+
+ public override int size()
+ {
+ return fields.Count;
+ }
+
+ public override long UniqueTermCount
+ {
+ get
+ {
+ long numTerms = 0;
+ foreach (DirectField field in fields.Values)
+ {
+ numTerms += field.terms.Length;
+ }
+ return numTerms;
+ }
+ }
+
+ public override void close()
+ {
+ }
+
+ public override long ramBytesUsed()
+ {
+ long sizeInBytes = 0;
+ foreach (KeyValuePair<string, DirectField> entry in fields.SetOfKeyValuePairs())
+ {
+ sizeInBytes += entry.Key.length() * RamUsageEstimator.NUM_BYTES_CHAR;
+ sizeInBytes += entry.Value.ramBytesUsed();
+ }
+ return sizeInBytes;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+ public override void checkIntegrity()
+ {
+ // if we read entirely into ram, we already validated.
+ // otherwise returned the raw postings reader
+ }
+ }
+
+ private sealed class DirectField : Terms
+ {
+
+ private abstract class TermAndSkip
+ {
+ public int[] skips;
+
+ /// <summary>
+ /// Returns the approximate number of RAM bytes used </summary>
+ public abstract long ramBytesUsed();
+ }
+
+ private sealed class LowFreqTerm : TermAndSkip
+ {
+ public readonly int[] postings;
+ public readonly sbyte[] payloads;
+ public readonly int docFreq;
+ public readonly int totalTermFreq;
+
+ public LowFreqTerm(int[] postings, sbyte[] payloads, int docFreq, int totalTermFreq)
+ {
+ this.postings = postings;
+ this.payloads = payloads;
+ this.docFreq = docFreq;
+ this.totalTermFreq = totalTermFreq;
+ }
+
+ public override long ramBytesUsed()
+ {
+ return ((postings != null) ? RamUsageEstimator.sizeOf(postings) : 0) + ((payloads != null) ? RamUsageEstimator.sizeOf(payloads) : 0);
+ }
+ }
+
+ // TODO: maybe specialize into prx/no-prx/no-frq cases?
+ private sealed class HighFreqTerm : TermAndSkip
+ {
+ public readonly long totalTermFreq;
+ public readonly int[] docIDs;
+ public readonly int[] freqs;
+ public readonly int[][] positions;
+ public readonly sbyte[][][] payloads;
+
+ public HighFreqTerm(int[] docIDs, int[] freqs, int[][] positions, sbyte[][][] payloads, long totalTermFreq)
+ {
+ this.docIDs = docIDs;
+ this.freqs = freqs;
+ this.positions = positions;
+ this.payloads = payloads;
+ this.totalTermFreq = totalTermFreq;
+ }
+
+ public override long ramBytesUsed()
+ {
+ long sizeInBytes = 0;
+ sizeInBytes += (docIDs != null)? RamUsageEstimator.sizeOf(docIDs) : 0;
+ sizeInBytes += (freqs != null)? RamUsageEstimator.sizeOf(freqs) : 0;
+
+ if (positions != null)
+ {
+ foreach (int[] position in positions)
+ {
+ sizeInBytes += (position != null) ? RamUsageEstimator.sizeOf(position) : 0;
+ }
+ }
+
+ if (payloads != null)
+ {
+ foreach (sbyte[][] payload in payloads)
+ {
+ if (payload != null)
+ {
+ foreach (sbyte[] pload in payload)
+ {
+ sizeInBytes += (pload != null) ? RamUsageEstimator.sizeOf(pload) : 0;
+ }
+ }
+ }
+ }
+
+ return sizeInBytes;
+ }
+ }
+
+ internal readonly sbyte[] termBytes;
+ internal readonly int[] termOffsets;
+
+ internal readonly int[] skips;
+ internal readonly int[] skipOffsets;
+
+ internal readonly TermAndSkip[] terms;
+ internal readonly bool hasFreq;
+ internal readonly bool hasPos;
+ internal readonly bool hasOffsets_Renamed;
+ internal readonly bool hasPayloads_Renamed;
+ internal readonly long sumTotalTermFreq;
+ internal readonly int docCount;
+ internal readonly long sumDocFreq;
+ internal int skipCount;
+
+ // TODO: maybe make a separate builder? These are only
+ // used during load:
+ internal int count;
+ internal int[] sameCounts = new int[10];
+ internal readonly int minSkipCount;
+
+ private sealed class IntArrayWriter
+ {
+ internal int[] ints = new int[10];
+ internal int upto;
+
+ public void add(int value)
+ {
+ if (ints.Length == upto)
+ {
+ ints = ArrayUtil.grow(ints);
+ }
+ ints[upto++] = value;
+ }
+
+ public int[] get()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int[] arr = new int[upto];
+ int[] arr = new int[upto];
+ Array.Copy(ints, 0, arr, 0, upto);
+ upto = 0;
+ return arr;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public DirectField(org.apache.lucene.index.SegmentReadState state, String field, org.apache.lucene.index.Terms termsIn, int minSkipCount, int lowFreqCutoff) throws java.io.IOException
+ public DirectField(SegmentReadState state, string field, Terms termsIn, int minSkipCount, int lowFreqCutoff)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
+ FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
+
+ sumTotalTermFreq = termsIn.SumTotalTermFreq;
+ sumDocFreq = termsIn.SumDocFreq;
+ docCount = termsIn.DocCount;
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int numTerms = (int) termsIn.size();
+ int numTerms = (int) termsIn.size();
+ if (numTerms == -1)
+ {
+ throw new System.ArgumentException("codec does not provide Terms.size()");
+ }
+ terms = new TermAndSkip[numTerms];
+ termOffsets = new int[1 + numTerms];
+
+ sbyte[] termBytes = new sbyte[1024];
+
+ this.minSkipCount = minSkipCount;
+
+ hasFreq = fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_ONLY) > 0;
+ hasPos = fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0;
+ hasOffsets_Renamed = fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) > 0;
+ hasPayloads_Renamed = fieldInfo.hasPayloads();
+
+ BytesRef term;
+ DocsEnum docsEnum = null;
+ DocsAndPositionsEnum docsAndPositionsEnum = null;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.TermsEnum termsEnum = termsIn.iterator(null);
+ TermsEnum termsEnum = termsIn.iterator(null);
+ int termOffset = 0;
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final IntArrayWriter scratch = new IntArrayWriter();
+ IntArrayWriter scratch = new IntArrayWriter();
+
+ // Used for payloads, if any:
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.store.RAMOutputStream ros = new org.apache.lucene.store.RAMOutputStream();
+ RAMOutputStream ros = new RAMOutputStream();
+
+ // if (DEBUG) {
+ // System.out.println("\nLOAD terms seg=" + state.segmentInfo.name + " field=" + field + " hasOffsets=" + hasOffsets + " hasFreq=" + hasFreq + " hasPos=" + hasPos + " hasPayloads=" + hasPayloads);
+ // }
+
+ while ((term = termsEnum.next()) != null)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int docFreq = termsEnum.docFreq();
+ int docFreq = termsEnum.docFreq();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long totalTermFreq = termsEnum.totalTermFreq();
+ long totalTermFreq = termsEnum.totalTermFreq();
+
+ // if (DEBUG) {
+ // System.out.println(" term=" + term.utf8ToString());
+ // }
+
+ termOffsets[count] = termOffset;
+
+ if (termBytes.Length < (termOffset + term.length))
+ {
+ termBytes = ArrayUtil.grow(termBytes, termOffset + term.length);
+ }
+ Array.Copy(term.bytes, term.offset, termBytes, termOffset, term.length);
+ termOffset += term.length;
+ termOffsets[count + 1] = termOffset;
+
+ if (hasPos)
+ {
+ docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum);
+ }
+ else
+ {
+ docsEnum = termsEnum.docs(null, docsEnum);
+ }
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TermAndSkip ent;
+ TermAndSkip ent;
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.DocsEnum docsEnum2;
+ DocsEnum docsEnum2;
+ if (hasPos)
+ {
+ docsEnum2 = docsAndPositionsEnum;
+ }
+ else
+ {
+ docsEnum2 = docsEnum;
+ }
+
+ int docID;
+
+ if (docFreq <= lowFreqCutoff)
+ {
+
+ ros.reset();
+
+ // Pack postings for low-freq terms into a single int[]:
+ while ((docID = docsEnum2.nextDoc()) != DocsEnum.NO_MORE_DOCS)
+ {
+ scratch.add(docID);
+ if (hasFreq)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int freq = docsEnum2.freq();
+ int freq = docsEnum2.freq();
+ scratch.add(freq);
+ if (hasPos)
+ {
+ for (int pos = 0;pos < freq;pos++)
+ {
+ scratch.add(docsAndPositionsEnum.nextPosition());
+ if (hasOffsets_Renamed)
+ {
+ scratch.add(docsAndPositionsEnum.startOffset());
+ scratch.add(docsAndPositionsEnum.endOffset());
+ }
+ if (hasPayloads_Renamed)
+ {
+//JAVA TO C# CONVERTER WARNING: The origina
<TRUNCATED>
[03/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs
index 90e4a53..f45bc24 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryPostingsFormat.cs
@@ -1,936 +1,1203 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import codecs.CodecUtil;
-import codecs.FieldsConsumer;
-import codecs.FieldsProducer;
-import codecs.PostingsConsumer;
-import codecs.PostingsFormat;
-import codecs.TermStats;
-import codecs.TermsConsumer;
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import index.Terms;
-import index.TermsEnum;
-import store.ByteArrayDataInput;
-import store.ChecksumIndexInput;
-import store.IOContext;
-import store.IndexInput;
-import store.IndexOutput;
-import store.RAMOutputStream;
-import util.ArrayUtil;
-import util.Bits;
-import util.BytesRef;
-import util.IOUtils;
-import util.IntsRef;
-import util.RamUsageEstimator;
-import util.fst.Builder;
-import util.fst.ByteSequenceOutputs;
-import util.fst.BytesRefFSTEnum;
-import util.fst.FST;
-import util.fst.Util;
-import util.packed.PackedInts;
-
-// TODO: would be nice to somehow allow this to act like
-// InstantiatedIndex, by never writing to disk; ie you write
-// to this Codec in RAM only and then when you open a reader
-// it pulls the FST directly from what you wrote w/o going
-// to disk.
-
-/** Stores terms & postings (docs, positions, payloads) in
- * RAM, using an FST.
- *
- * <p>Note that this codec implements advance as a linear
- * scan! This means if you store large fields in here,
- * queries that rely on advance will (AND boolQuery,
- * PhraseQuery) will be relatively slow!
- *
- * @lucene.experimental */
-
-// TODO: Maybe name this 'Cached' or something to reflect
-// the reality that it is actually written to disk, but
-// loads itself in ram?
-public final class MemoryPostingsFormat extends PostingsFormat {
-
- private final bool doPackFST;
- private final float acceptableOverheadRatio;
-
- public MemoryPostingsFormat() {
- this(false, PackedInts.DEFAULT);
- }
-
- /**
- * Create MemoryPostingsFormat, specifying advanced FST options.
- * @param doPackFST true if a packed FST should be built.
- * NOTE: packed FSTs are limited to ~2.1 GB of postings.
- * @param acceptableOverheadRatio allowable overhead for packed ints
- * during FST construction.
- */
- public MemoryPostingsFormat(bool doPackFST, float acceptableOverheadRatio) {
- super("Memory");
- this.doPackFST = doPackFST;
- this.acceptableOverheadRatio = acceptableOverheadRatio;
- }
-
- @Override
- public String toString() {
- return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")";
- }
-
- private final static class TermsWriter extends TermsConsumer {
- private final IndexOutput out;
- private final FieldInfo field;
- private final Builder<BytesRef> builder;
- private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
- private final bool doPackFST;
- private final float acceptableOverheadRatio;
- private int termCount;
-
- public TermsWriter(IndexOutput out, FieldInfo field, bool doPackFST, float acceptableOverheadRatio) {
- this.out = out;
- this.field = field;
- this.doPackFST = doPackFST;
- this.acceptableOverheadRatio = acceptableOverheadRatio;
- builder = new Builder<>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio, true, 15);
- }
-
- private class PostingsWriter extends PostingsConsumer {
- private int lastDocID;
- private int lastPos;
- private int lastPayloadLen;
-
- // NOTE: not private so we don't pay access check at runtime:
- int docCount;
- RAMOutputStream buffer = new RAMOutputStream();
-
- int lastOffsetLength;
- int lastOffset;
-
- @Override
- public void startDoc(int docID, int termDocFreq) {
- //System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq);
- final int delta = docID - lastDocID;
- Debug.Assert( docID == 0 || delta > 0;
- lastDocID = docID;
- docCount++;
-
- if (field.getIndexOptions() == IndexOptions.DOCS_ONLY) {
- buffer.writeVInt(delta);
- } else if (termDocFreq == 1) {
- buffer.writeVInt((delta<<1) | 1);
- } else {
- buffer.writeVInt(delta<<1);
- Debug.Assert( termDocFreq > 0;
- buffer.writeVInt(termDocFreq);
- }
-
- lastPos = 0;
- lastOffset = 0;
- }
-
- @Override
- public void addPosition(int pos, BytesRef payload, int startOffset, int endOffset) {
- Debug.Assert( payload == null || field.hasPayloads();
-
- //System.out.println(" addPos pos=" + pos + " payload=" + payload);
-
- final int delta = pos - lastPos;
- Debug.Assert( delta >= 0;
- lastPos = pos;
-
- int payloadLen = 0;
-
- if (field.hasPayloads()) {
- payloadLen = payload == null ? 0 : payload.length;
- if (payloadLen != lastPayloadLen) {
- lastPayloadLen = payloadLen;
- buffer.writeVInt((delta<<1)|1);
- buffer.writeVInt(payloadLen);
- } else {
- buffer.writeVInt(delta<<1);
- }
- } else {
- buffer.writeVInt(delta);
- }
-
- if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
- // don't use startOffset - lastEndOffset, because this creates lots of negative vints for synonyms,
- // and the numbers aren't that much smaller anyways.
- int offsetDelta = startOffset - lastOffset;
- int offsetLength = endOffset - startOffset;
- if (offsetLength != lastOffsetLength) {
- buffer.writeVInt(offsetDelta << 1 | 1);
- buffer.writeVInt(offsetLength);
- } else {
- buffer.writeVInt(offsetDelta << 1);
- }
- lastOffset = startOffset;
- lastOffsetLength = offsetLength;
- }
-
- if (payloadLen > 0) {
- buffer.writeBytes(payload.bytes, payload.offset, payloadLen);
- }
- }
-
- @Override
- public void finishDoc() {
- }
-
- public PostingsWriter reset() {
- Debug.Assert( buffer.getFilePointer() == 0;
- lastDocID = 0;
- docCount = 0;
- lastPayloadLen = 0;
- // force first offset to write its length
- lastOffsetLength = -1;
- return this;
- }
- }
-
- private final PostingsWriter postingsWriter = new PostingsWriter();
-
- @Override
- public PostingsConsumer startTerm(BytesRef text) {
- //System.out.println(" startTerm term=" + text.utf8ToString());
- return postingsWriter.reset();
- }
-
- private final RAMOutputStream buffer2 = new RAMOutputStream();
- private final BytesRef spare = new BytesRef();
- private byte[] finalBuffer = new byte[128];
-
- private final IntsRef scratchIntsRef = new IntsRef();
-
- @Override
- public void finishTerm(BytesRef text, TermStats stats) {
-
- Debug.Assert( postingsWriter.docCount == stats.docFreq;
-
- Debug.Assert( buffer2.getFilePointer() == 0;
-
- buffer2.writeVInt(stats.docFreq);
- if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- buffer2.writeVLong(stats.totalTermFreq-stats.docFreq);
- }
- int pos = (int) buffer2.getFilePointer();
- buffer2.writeTo(finalBuffer, 0);
- buffer2.reset();
-
- final int totalBytes = pos + (int) postingsWriter.buffer.getFilePointer();
- if (totalBytes > finalBuffer.length) {
- finalBuffer = ArrayUtil.grow(finalBuffer, totalBytes);
- }
- postingsWriter.buffer.writeTo(finalBuffer, pos);
- postingsWriter.buffer.reset();
-
- spare.bytes = finalBuffer;
- spare.length = totalBytes;
-
- //System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq);
- //for(int i=0;i<totalBytes;i++) {
- // System.out.println(" " + Integer.toHexString(finalBuffer[i]&0xFF));
- //}
-
- builder.add(Util.toIntsRef(text, scratchIntsRef), BytesRef.deepCopyOf(spare));
- termCount++;
- }
-
- @Override
- public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) {
- if (termCount > 0) {
- out.writeVInt(termCount);
- out.writeVInt(field.number);
- if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- out.writeVLong(sumTotalTermFreq);
- }
- out.writeVLong(sumDocFreq);
- out.writeVInt(docCount);
- FST<BytesRef> fst = builder.finish();
- fst.save(out);
- //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
- }
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
- }
-
- private static String EXTENSION = "ram";
- private static final String CODEC_NAME = "MemoryPostings";
- private static final int VERSION_START = 0;
- private static final int VERSION_CURRENT = VERSION_START;
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
-
- final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- final IndexOutput out = state.directory.createOutput(fileName, state.context);
- bool success = false;
- try {
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(out);
- }
- }
-
- return new FieldsConsumer() {
- @Override
- public TermsConsumer addField(FieldInfo field) {
- //System.out.println("\naddField field=" + field.name);
- return new TermsWriter(out, field, doPackFST, acceptableOverheadRatio);
- }
-
- @Override
- public void close() {
- // EOF marker:
- try {
- out.writeVInt(0);
- CodecUtil.writeFooter(out);
- } finally {
- out.close();
- }
- }
- };
- }
-
- private final static class FSTDocsEnum extends DocsEnum {
- private final IndexOptions indexOptions;
- private final bool storePayloads;
- private byte[] buffer = new byte[16];
- private final ByteArrayDataInput in = new ByteArrayDataInput(buffer);
-
- private Bits liveDocs;
- private int docUpto;
- private int docID = -1;
- private int accum;
- private int freq;
- private int payloadLen;
- private int numDocs;
-
- public FSTDocsEnum(IndexOptions indexOptions, bool storePayloads) {
- this.indexOptions = indexOptions;
- this.storePayloads = storePayloads;
- }
-
- public bool canReuse(IndexOptions indexOptions, bool storePayloads) {
- return indexOptions == this.indexOptions && storePayloads == this.storePayloads;
- }
-
- public FSTDocsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) {
- Debug.Assert( numDocs > 0;
- if (buffer.length < bufferIn.length) {
- buffer = ArrayUtil.grow(buffer, bufferIn.length);
- }
- in.reset(buffer, 0, bufferIn.length);
- System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length);
- this.liveDocs = liveDocs;
- docID = -1;
- accum = 0;
- docUpto = 0;
- freq = 1;
- payloadLen = 0;
- this.numDocs = numDocs;
- return this;
- }
-
- @Override
- public int nextDoc() {
- while(true) {
- //System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
- if (docUpto == numDocs) {
- // System.out.println(" END");
- return docID = NO_MORE_DOCS;
- }
- docUpto++;
- if (indexOptions == IndexOptions.DOCS_ONLY) {
- accum += in.readVInt();
- } else {
- final int code = in.readVInt();
- accum += code >>> 1;
- //System.out.println(" docID=" + accum + " code=" + code);
- if ((code & 1) != 0) {
- freq = 1;
- } else {
- freq = in.readVInt();
- Debug.Assert( freq > 0;
- }
-
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- // Skip positions/payloads
- for(int posUpto=0;posUpto<freq;posUpto++) {
- if (!storePayloads) {
- in.readVInt();
- } else {
- final int posCode = in.readVInt();
- if ((posCode & 1) != 0) {
- payloadLen = in.readVInt();
- }
- in.skipBytes(payloadLen);
- }
- }
- } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
- // Skip positions/offsets/payloads
- for(int posUpto=0;posUpto<freq;posUpto++) {
- int posCode = in.readVInt();
- if (storePayloads && ((posCode & 1) != 0)) {
- payloadLen = in.readVInt();
- }
- if ((in.readVInt() & 1) != 0) {
- // new offset length
- in.readVInt();
- }
- if (storePayloads) {
- in.skipBytes(payloadLen);
- }
- }
- }
- }
-
- if (liveDocs == null || liveDocs.get(accum)) {
- //System.out.println(" return docID=" + accum + " freq=" + freq);
- return (docID = accum);
- }
- }
- }
-
- @Override
- public int docID() {
- return docID;
- }
-
- @Override
- public int advance(int target) {
- // TODO: we could make more efficient version, but, it
- // should be rare that this will matter in practice
- // since usually apps will not store "big" fields in
- // this codec!
- return slowAdvance(target);
- }
-
- @Override
- public int freq() {
- return freq;
- }
-
- @Override
- public long cost() {
- return numDocs;
- }
- }
-
- private final static class FSTDocsAndPositionsEnum extends DocsAndPositionsEnum {
- private final bool storePayloads;
- private byte[] buffer = new byte[16];
- private final ByteArrayDataInput in = new ByteArrayDataInput(buffer);
-
- private Bits liveDocs;
- private int docUpto;
- private int docID = -1;
- private int accum;
- private int freq;
- private int numDocs;
- private int posPending;
- private int payloadLength;
- final bool storeOffsets;
- int offsetLength;
- int startOffset;
-
- private int pos;
- private final BytesRef payload = new BytesRef();
-
- public FSTDocsAndPositionsEnum(bool storePayloads, bool storeOffsets) {
- this.storePayloads = storePayloads;
- this.storeOffsets = storeOffsets;
- }
-
- public bool canReuse(bool storePayloads, bool storeOffsets) {
- return storePayloads == this.storePayloads && storeOffsets == this.storeOffsets;
- }
-
- public FSTDocsAndPositionsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) {
- Debug.Assert( numDocs > 0;
-
- // System.out.println("D&P reset bytes this=" + this);
- // for(int i=bufferIn.offset;i<bufferIn.length;i++) {
- // System.out.println(" " + Integer.toHexString(bufferIn.bytes[i]&0xFF));
- // }
-
- if (buffer.length < bufferIn.length) {
- buffer = ArrayUtil.grow(buffer, bufferIn.length);
- }
- in.reset(buffer, 0, bufferIn.length - bufferIn.offset);
- System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length);
- this.liveDocs = liveDocs;
- docID = -1;
- accum = 0;
- docUpto = 0;
- payload.bytes = buffer;
- payloadLength = 0;
- this.numDocs = numDocs;
- posPending = 0;
- startOffset = storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
- offsetLength = 0;
- return this;
- }
-
- @Override
- public int nextDoc() {
- while (posPending > 0) {
- nextPosition();
- }
- while(true) {
- //System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
- if (docUpto == numDocs) {
- //System.out.println(" END");
- return docID = NO_MORE_DOCS;
- }
- docUpto++;
-
- final int code = in.readVInt();
- accum += code >>> 1;
- if ((code & 1) != 0) {
- freq = 1;
- } else {
- freq = in.readVInt();
- Debug.Assert( freq > 0;
- }
-
- if (liveDocs == null || liveDocs.get(accum)) {
- pos = 0;
- startOffset = storeOffsets ? 0 : -1;
- posPending = freq;
- //System.out.println(" return docID=" + accum + " freq=" + freq);
- return (docID = accum);
- }
-
- // Skip positions
- for(int posUpto=0;posUpto<freq;posUpto++) {
- if (!storePayloads) {
- in.readVInt();
- } else {
- final int skipCode = in.readVInt();
- if ((skipCode & 1) != 0) {
- payloadLength = in.readVInt();
- //System.out.println(" new payloadLen=" + payloadLength);
- }
- }
-
- if (storeOffsets) {
- if ((in.readVInt() & 1) != 0) {
- // new offset length
- offsetLength = in.readVInt();
- }
- }
-
- if (storePayloads) {
- in.skipBytes(payloadLength);
- }
- }
- }
- }
-
- @Override
- public int nextPosition() {
- //System.out.println(" nextPos storePayloads=" + storePayloads + " this=" + this);
- Debug.Assert( posPending > 0;
- posPending--;
- if (!storePayloads) {
- pos += in.readVInt();
- } else {
- final int code = in.readVInt();
- pos += code >>> 1;
- if ((code & 1) != 0) {
- payloadLength = in.readVInt();
- //System.out.println(" new payloadLen=" + payloadLength);
- //} else {
- //System.out.println(" same payloadLen=" + payloadLength);
- }
- }
-
- if (storeOffsets) {
- int offsetCode = in.readVInt();
- if ((offsetCode & 1) != 0) {
- // new offset length
- offsetLength = in.readVInt();
- }
- startOffset += offsetCode >>> 1;
- }
-
- if (storePayloads) {
- payload.offset = in.getPosition();
- in.skipBytes(payloadLength);
- payload.length = payloadLength;
- }
-
- //System.out.println(" pos=" + pos + " payload=" + payload + " fp=" + in.getPosition());
- return pos;
- }
-
- @Override
- public int startOffset() {
- return startOffset;
- }
-
- @Override
- public int endOffset() {
- return startOffset + offsetLength;
- }
-
- @Override
- public BytesRef getPayload() {
- return payload.length > 0 ? payload : null;
- }
-
- @Override
- public int docID() {
- return docID;
- }
-
- @Override
- public int advance(int target) {
- // TODO: we could make more efficient version, but, it
- // should be rare that this will matter in practice
- // since usually apps will not store "big" fields in
- // this codec!
- return slowAdvance(target);
- }
-
- @Override
- public int freq() {
- return freq;
- }
-
- @Override
- public long cost() {
- return numDocs;
- }
- }
-
- private final static class FSTTermsEnum extends TermsEnum {
- private final FieldInfo field;
- private final BytesRefFSTEnum<BytesRef> fstEnum;
- private final ByteArrayDataInput buffer = new ByteArrayDataInput();
- private bool didDecode;
-
- private int docFreq;
- private long totalTermFreq;
- private BytesRefFSTEnum.InputOutput<BytesRef> current;
- private BytesRef postingsSpare = new BytesRef();
-
- public FSTTermsEnum(FieldInfo field, FST<BytesRef> fst) {
- this.field = field;
- fstEnum = new BytesRefFSTEnum<>(fst);
- }
-
- private void decodeMetaData() {
- if (!didDecode) {
- buffer.reset(current.output.bytes, current.output.offset, current.output.length);
- docFreq = buffer.readVInt();
- if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- totalTermFreq = docFreq + buffer.readVLong();
- } else {
- totalTermFreq = -1;
- }
- postingsSpare.bytes = current.output.bytes;
- postingsSpare.offset = buffer.getPosition();
- postingsSpare.length = current.output.length - (buffer.getPosition() - current.output.offset);
- //System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length);
- didDecode = true;
- }
- }
-
- @Override
- public bool seekExact(BytesRef text) {
- //System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
- current = fstEnum.seekExact(text);
- didDecode = false;
- return current != null;
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef text) {
- //System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
- current = fstEnum.seekCeil(text);
- if (current == null) {
- return SeekStatus.END;
- } else {
-
- // System.out.println(" got term=" + current.input.utf8ToString());
- // for(int i=0;i<current.output.length;i++) {
- // System.out.println(" " + Integer.toHexString(current.output.bytes[i]&0xFF));
- // }
-
- didDecode = false;
-
- if (text.equals(current.input)) {
- //System.out.println(" found!");
- return SeekStatus.FOUND;
- } else {
- //System.out.println(" not found: " + current.input.utf8ToString());
- return SeekStatus.NOT_FOUND;
- }
- }
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
- decodeMetaData();
- FSTDocsEnum docsEnum;
-
- if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
- docsEnum = new FSTDocsEnum(field.getIndexOptions(), field.hasPayloads());
- } else {
- docsEnum = (FSTDocsEnum) reuse;
- if (!docsEnum.canReuse(field.getIndexOptions(), field.hasPayloads())) {
- docsEnum = new FSTDocsEnum(field.getIndexOptions(), field.hasPayloads());
- }
- }
- return docsEnum.reset(this.postingsSpare, liveDocs, docFreq);
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
-
- bool hasOffsets = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
- return null;
- }
- decodeMetaData();
- FSTDocsAndPositionsEnum docsAndPositionsEnum;
- if (reuse == null || !(reuse instanceof FSTDocsAndPositionsEnum)) {
- docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.hasPayloads(), hasOffsets);
- } else {
- docsAndPositionsEnum = (FSTDocsAndPositionsEnum) reuse;
- if (!docsAndPositionsEnum.canReuse(field.hasPayloads(), hasOffsets)) {
- docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.hasPayloads(), hasOffsets);
- }
- }
- //System.out.println("D&P reset this=" + this);
- return docsAndPositionsEnum.reset(postingsSpare, liveDocs, docFreq);
- }
-
- @Override
- public BytesRef term() {
- return current.input;
- }
-
- @Override
- public BytesRef next() {
- //System.out.println("te.next");
- current = fstEnum.next();
- if (current == null) {
- //System.out.println(" END");
- return null;
- }
- didDecode = false;
- //System.out.println(" term=" + field.name + ":" + current.input.utf8ToString());
- return current.input;
- }
-
- @Override
- public int docFreq() {
- decodeMetaData();
- return docFreq;
- }
-
- @Override
- public long totalTermFreq() {
- decodeMetaData();
- return totalTermFreq;
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public void seekExact(long ord) {
- // NOTE: we could add this...
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long ord() {
- // NOTE: we could add this...
- throw new UnsupportedOperationException();
- }
- }
-
- private final static class TermsReader extends Terms {
-
- private final long sumTotalTermFreq;
- private final long sumDocFreq;
- private final int docCount;
- private final int termCount;
- private FST<BytesRef> fst;
- private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
- private final FieldInfo field;
-
- public TermsReader(FieldInfos fieldInfos, IndexInput in, int termCount) {
- this.termCount = termCount;
- final int fieldNumber = in.readVInt();
- field = fieldInfos.fieldInfo(fieldNumber);
- if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- sumTotalTermFreq = in.readVLong();
- } else {
- sumTotalTermFreq = -1;
- }
- sumDocFreq = in.readVLong();
- docCount = in.readVInt();
-
- fst = new FST<>(in, outputs);
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return sumTotalTermFreq;
- }
-
- @Override
- public long getSumDocFreq() {
- return sumDocFreq;
- }
-
- @Override
- public int getDocCount() {
- return docCount;
- }
-
- @Override
- public long size() {
- return termCount;
- }
-
- @Override
- public TermsEnum iterator(TermsEnum reuse) {
- return new FSTTermsEnum(field, fst);
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public bool hasFreqs() {
- return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- }
-
- @Override
- public bool hasOffsets() {
- return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
-
- @Override
- public bool hasPositions() {
- return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
-
- @Override
- public bool hasPayloads() {
- return field.hasPayloads();
- }
-
- public long ramBytesUsed() {
- return ((fst!=null) ? fst.sizeInBytes() : 0);
- }
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) {
- final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- final ChecksumIndexInput in = state.directory.openChecksumInput(fileName, IOContext.READONCE);
-
- final SortedMap<String,TermsReader> fields = new TreeMap<>();
-
- try {
- CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
- while(true) {
- final int termCount = in.readVInt();
- if (termCount == 0) {
- break;
- }
- final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
- // System.out.println("load field=" + termsReader.field.name);
- fields.put(termsReader.field.name, termsReader);
- }
- CodecUtil.checkFooter(in);
- } finally {
- in.close();
- }
-
- return new FieldsProducer() {
- @Override
- public Iterator<String> iterator() {
- return Collections.unmodifiableSet(fields.keySet()).iterator();
- }
-
- @Override
- public Terms terms(String field) {
- return fields.get(field);
- }
-
- @Override
- public int size() {
- return fields.size();
- }
-
- @Override
- public void close() {
- // Drop ref to FST:
- for(TermsReader termsReader : fields.values()) {
- termsReader.fst = null;
- }
- }
-
- @Override
- public long ramBytesUsed() {
- long sizeInBytes = 0;
- for(Map.Entry<String,TermsReader> entry: fields.entrySet()) {
- sizeInBytes += (entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR);
- sizeInBytes += entry.getValue().ramBytesUsed();
- }
- return sizeInBytes;
- }
-
- @Override
- public void checkIntegrity() {}
- };
- }
-}
+using System;
+using System.Diagnostics;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using DocsAndPositionsEnum = org.apache.lucene.index.DocsAndPositionsEnum;
+ using DocsEnum = org.apache.lucene.index.DocsEnum;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using FieldInfos = org.apache.lucene.index.FieldInfos;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using Terms = org.apache.lucene.index.Terms;
+ using TermsEnum = org.apache.lucene.index.TermsEnum;
+ using ByteArrayDataInput = org.apache.lucene.store.ByteArrayDataInput;
+ using ChecksumIndexInput = org.apache.lucene.store.ChecksumIndexInput;
+ using IOContext = org.apache.lucene.store.IOContext;
+ using IndexInput = org.apache.lucene.store.IndexInput;
+ using IndexOutput = org.apache.lucene.store.IndexOutput;
+ using RAMOutputStream = org.apache.lucene.store.RAMOutputStream;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using Bits = org.apache.lucene.util.Bits;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using IntsRef = org.apache.lucene.util.IntsRef;
+ using RamUsageEstimator = org.apache.lucene.util.RamUsageEstimator;
+ using Builder = org.apache.lucene.util.fst.Builder;
+ using ByteSequenceOutputs = org.apache.lucene.util.fst.ByteSequenceOutputs;
+ using BytesRefFSTEnum = org.apache.lucene.util.fst.BytesRefFSTEnum;
+ using FST = org.apache.lucene.util.fst.FST;
+ using Util = org.apache.lucene.util.fst.Util;
+ using PackedInts = org.apache.lucene.util.packed.PackedInts;
+
+ // TODO: would be nice to somehow allow this to act like
+ // InstantiatedIndex, by never writing to disk; ie you write
+ // to this Codec in RAM only and then when you open a reader
+ // it pulls the FST directly from what you wrote w/o going
+ // to disk.
+
+ /// <summary>
+ /// Stores terms & postings (docs, positions, payloads) in
+ /// RAM, using an FST.
+ ///
+ /// <para>Note that this codec implements advance as a linear
+ /// scan! This means if you store large fields in here,
+ /// queries that rely on advance will (AND BooleanQuery,
+ /// PhraseQuery) will be relatively slow!
+ ///
+ /// @lucene.experimental
+ /// </para>
+ /// </summary>
+
+ // TODO: Maybe name this 'Cached' or something to reflect
+ // the reality that it is actually written to disk, but
+ // loads itself in ram?
+ public sealed class MemoryPostingsFormat : PostingsFormat
+ {
+
+ private readonly bool doPackFST;
+ private readonly float acceptableOverheadRatio;
+
+ public MemoryPostingsFormat() : this(false, PackedInts.DEFAULT)
+ {
+ }
+
+ /// <summary>
+ /// Create MemoryPostingsFormat, specifying advanced FST options. </summary>
+ /// <param name="doPackFST"> true if a packed FST should be built.
+ /// NOTE: packed FSTs are limited to ~2.1 GB of postings. </param>
+ /// <param name="acceptableOverheadRatio"> allowable overhead for packed ints
+ /// during FST construction. </param>
+ public MemoryPostingsFormat(bool doPackFST, float acceptableOverheadRatio) : base("Memory")
+ {
+ this.doPackFST = doPackFST;
+ this.acceptableOverheadRatio = acceptableOverheadRatio;
+ }
+
+ public override string ToString()
+ {
+ return "PostingsFormat(name=" + Name + " doPackFST= " + doPackFST + ")";
+ }
+
+ private sealed class TermsWriter : TermsConsumer
+ {
+ internal bool InstanceFieldsInitialized = false;
+
+ internal void InitializeInstanceFields()
+ {
+ postingsWriter = new PostingsWriter(this);
+ }
+
+ internal readonly IndexOutput @out;
+ internal readonly FieldInfo field;
+ internal readonly Builder<BytesRef> builder;
+ internal readonly ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton;
+ internal readonly bool doPackFST;
+ internal readonly float acceptableOverheadRatio;
+ internal int termCount;
+
+ public TermsWriter(IndexOutput @out, FieldInfo field, bool doPackFST, float acceptableOverheadRatio)
+ {
+ if (!InstanceFieldsInitialized)
+ {
+ InitializeInstanceFields();
+ InstanceFieldsInitialized = true;
+ }
+ this.@out = @out;
+ this.field = field;
+ this.doPackFST = doPackFST;
+ this.acceptableOverheadRatio = acceptableOverheadRatio;
+ builder = new Builder<>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, doPackFST, acceptableOverheadRatio, true, 15);
+ }
+
+ private class PostingsWriter : PostingsConsumer
+ {
+ private readonly MemoryPostingsFormat.TermsWriter outerInstance;
+
+ public PostingsWriter(MemoryPostingsFormat.TermsWriter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ internal int lastDocID;
+ internal int lastPos;
+ internal int lastPayloadLen;
+
+ // NOTE: not private so we don't pay access check at runtime:
+ internal int docCount;
+ internal RAMOutputStream buffer = new RAMOutputStream();
+
+ internal int lastOffsetLength;
+ internal int lastOffset;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void startDoc(int docID, int termDocFreq) throws java.io.IOException
+ public override void startDoc(int docID, int termDocFreq)
+ {
+ //System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int delta = docID - lastDocID;
+ int delta = docID - lastDocID;
+ Debug.Assert(docID == 0 || delta > 0);
+ lastDocID = docID;
+ docCount++;
+
+ if (outerInstance.field.IndexOptions == IndexOptions.DOCS_ONLY)
+ {
+ buffer.writeVInt(delta);
+ }
+ else if (termDocFreq == 1)
+ {
+ buffer.writeVInt((delta << 1) | 1);
+ }
+ else
+ {
+ buffer.writeVInt(delta << 1);
+ Debug.Assert(termDocFreq > 0);
+ buffer.writeVInt(termDocFreq);
+ }
+
+ lastPos = 0;
+ lastOffset = 0;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addPosition(int pos, org.apache.lucene.util.BytesRef payload, int startOffset, int endOffset) throws java.io.IOException
+ public override void addPosition(int pos, BytesRef payload, int startOffset, int endOffset)
+ {
+ Debug.Assert(payload == null || outerInstance.field.hasPayloads());
+
+ //System.out.println(" addPos pos=" + pos + " payload=" + payload);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int delta = pos - lastPos;
+ int delta = pos - lastPos;
+ Debug.Assert(delta >= 0);
+ lastPos = pos;
+
+ int payloadLen = 0;
+
+ if (outerInstance.field.hasPayloads())
+ {
+ payloadLen = payload == null ? 0 : payload.length;
+ if (payloadLen != lastPayloadLen)
+ {
+ lastPayloadLen = payloadLen;
+ buffer.writeVInt((delta << 1) | 1);
+ buffer.writeVInt(payloadLen);
+ }
+ else
+ {
+ buffer.writeVInt(delta << 1);
+ }
+ }
+ else
+ {
+ buffer.writeVInt(delta);
+ }
+
+ if (outerInstance.field.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0)
+ {
+ // don't use startOffset - lastEndOffset, because this creates lots of negative vints for synonyms,
+ // and the numbers aren't that much smaller anyways.
+ int offsetDelta = startOffset - lastOffset;
+ int offsetLength = endOffset - startOffset;
+ if (offsetLength != lastOffsetLength)
+ {
+ buffer.writeVInt(offsetDelta << 1 | 1);
+ buffer.writeVInt(offsetLength);
+ }
+ else
+ {
+ buffer.writeVInt(offsetDelta << 1);
+ }
+ lastOffset = startOffset;
+ lastOffsetLength = offsetLength;
+ }
+
+ if (payloadLen > 0)
+ {
+ buffer.writeBytes(payload.bytes, payload.offset, payloadLen);
+ }
+ }
+
+ public override void finishDoc()
+ {
+ }
+
+ public virtual PostingsWriter reset()
+ {
+ Debug.Assert(buffer.FilePointer == 0);
+ lastDocID = 0;
+ docCount = 0;
+ lastPayloadLen = 0;
+ // force first offset to write its length
+ lastOffsetLength = -1;
+ return this;
+ }
+ }
+
+ internal PostingsWriter postingsWriter;
+
+ public override PostingsConsumer startTerm(BytesRef text)
+ {
+ //System.out.println(" startTerm term=" + text.utf8ToString());
+ return postingsWriter.reset();
+ }
+
+ internal readonly RAMOutputStream buffer2 = new RAMOutputStream();
+ internal readonly BytesRef spare = new BytesRef();
+ internal sbyte[] finalBuffer = new sbyte[128];
+
+ internal readonly IntsRef scratchIntsRef = new IntsRef();
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void finishTerm(org.apache.lucene.util.BytesRef text, org.apache.lucene.codecs.TermStats stats) throws java.io.IOException
+ public override void finishTerm(BytesRef text, TermStats stats)
+ {
+
+ Debug.Assert(postingsWriter.docCount == stats.docFreq);
+
+ Debug.Assert(buffer2.FilePointer == 0);
+
+ buffer2.writeVInt(stats.docFreq);
+ if (field.IndexOptions != IndexOptions.DOCS_ONLY)
+ {
+ buffer2.writeVLong(stats.totalTermFreq - stats.docFreq);
+ }
+ int pos = (int) buffer2.FilePointer;
+ buffer2.writeTo(finalBuffer, 0);
+ buffer2.reset();
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int totalBytes = pos + (int) postingsWriter.buffer.getFilePointer();
+ int totalBytes = pos + (int) postingsWriter.buffer.FilePointer;
+ if (totalBytes > finalBuffer.Length)
+ {
+ finalBuffer = ArrayUtil.grow(finalBuffer, totalBytes);
+ }
+ postingsWriter.buffer.writeTo(finalBuffer, pos);
+ postingsWriter.buffer.reset();
+
+ spare.bytes = finalBuffer;
+ spare.length = totalBytes;
+
+ //System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq);
+ //for(int i=0;i<totalBytes;i++) {
+ // System.out.println(" " + Integer.toHexString(finalBuffer[i]&0xFF));
+ //}
+
+ builder.add(Util.toIntsRef(text, scratchIntsRef), BytesRef.deepCopyOf(spare));
+ termCount++;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws java.io.IOException
+ public override void finish(long sumTotalTermFreq, long sumDocFreq, int docCount)
+ {
+ if (termCount > 0)
+ {
+ @out.writeVInt(termCount);
+ @out.writeVInt(field.number);
+ if (field.IndexOptions != IndexOptions.DOCS_ONLY)
+ {
+ @out.writeVLong(sumTotalTermFreq);
+ }
+ @out.writeVLong(sumDocFreq);
+ @out.writeVInt(docCount);
+ FST<BytesRef> fst = builder.finish();
+ fst.save(@out);
+ //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
+ }
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+ }
+
+ private static string EXTENSION = "ram";
+ private const string CODEC_NAME = "MemoryPostings";
+ private const int VERSION_START = 0;
+ private const int VERSION_CURRENT = VERSION_START;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ {
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String fileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+ string fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.store.IndexOutput out = state.directory.createOutput(fileName, state.context);
+ IndexOutput @out = state.directory.createOutput(fileName, state.context);
+ bool success = false;
+ try
+ {
+ CodecUtil.writeHeader(@out, CODEC_NAME, VERSION_CURRENT);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(@out);
+ }
+ }
+
+ return new FieldsConsumerAnonymousInnerClassHelper(this, @out);
+ }
+
+ private class FieldsConsumerAnonymousInnerClassHelper : FieldsConsumer
+ {
+ private readonly MemoryPostingsFormat outerInstance;
+
+ private IndexOutput @out;
+
+ public FieldsConsumerAnonymousInnerClassHelper(MemoryPostingsFormat outerInstance, IndexOutput @out)
+ {
+ this.outerInstance = outerInstance;
+ this.@out = @out;
+ }
+
+ public override TermsConsumer addField(FieldInfo field)
+ {
+ //System.out.println("\naddField field=" + field.name);
+ return new TermsWriter(@out, field, outerInstance.doPackFST, outerInstance.acceptableOverheadRatio);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ // EOF marker:
+ try
+ {
+ @out.writeVInt(0);
+ CodecUtil.writeFooter(@out);
+ }
+ finally
+ {
+ @out.close();
+ }
+ }
+ }
+
+ private sealed class FSTDocsEnum : DocsEnum
+ {
+ internal bool InstanceFieldsInitialized = false;
+
+ internal void InitializeInstanceFields()
+ {
+ @in = new ByteArrayDataInput(buffer);
+ }
+
+ internal readonly IndexOptions indexOptions;
+ internal readonly bool storePayloads;
+ internal sbyte[] buffer = new sbyte[16];
+ internal ByteArrayDataInput @in;
+
+ internal Bits liveDocs;
+ internal int docUpto;
+ internal int docID_Renamed = -1;
+ internal int accum;
+ internal int freq_Renamed;
+ internal int payloadLen;
+ internal int numDocs;
+
+ public FSTDocsEnum(IndexOptions indexOptions, bool storePayloads)
+ {
+ if (!InstanceFieldsInitialized)
+ {
+ InitializeInstanceFields();
+ InstanceFieldsInitialized = true;
+ }
+ this.indexOptions = indexOptions;
+ this.storePayloads = storePayloads;
+ }
+
+ public bool canReuse(IndexOptions indexOptions, bool storePayloads)
+ {
+ return indexOptions == this.indexOptions && storePayloads == this.storePayloads;
+ }
+
+ public FSTDocsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs)
+ {
+ Debug.Assert(numDocs > 0);
+ if (buffer.Length < bufferIn.length)
+ {
+ buffer = ArrayUtil.grow(buffer, bufferIn.length);
+ }
+ @in.reset(buffer, 0, bufferIn.length);
+ Array.Copy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length);
+ this.liveDocs = liveDocs;
+ docID_Renamed = -1;
+ accum = 0;
+ docUpto = 0;
+ freq_Renamed = 1;
+ payloadLen = 0;
+ this.numDocs = numDocs;
+ return this;
+ }
+
+ public override int nextDoc()
+ {
+ while (true)
+ {
+ //System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
+ if (docUpto == numDocs)
+ {
+ // System.out.println(" END");
+ return docID_Renamed = NO_MORE_DOCS;
+ }
+ docUpto++;
+ if (indexOptions == IndexOptions.DOCS_ONLY)
+ {
+ accum += @in.readVInt();
+ }
+ else
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int code = in.readVInt();
+ int code = @in.readVInt();
+ accum += (int)((uint)code >> 1);
+ //System.out.println(" docID=" + accum + " code=" + code);
+ if ((code & 1) != 0)
+ {
+ freq_Renamed = 1;
+ }
+ else
+ {
+ freq_Renamed = @in.readVInt();
+ Debug.Assert(freq_Renamed > 0);
+ }
+
+ if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ {
+ // Skip positions/payloads
+ for (int posUpto = 0;posUpto < freq_Renamed;posUpto++)
+ {
+ if (!storePayloads)
+ {
+ @in.readVInt();
+ }
+ else
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int posCode = in.readVInt();
+ int posCode = @in.readVInt();
+ if ((posCode & 1) != 0)
+ {
+ payloadLen = @in.readVInt();
+ }
+ @in.skipBytes(payloadLen);
+ }
+ }
+ }
+ else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ {
+ // Skip positions/offsets/payloads
+ for (int posUpto = 0;posUpto < freq_Renamed;posUpto++)
+ {
+ int posCode = @in.readVInt();
+ if (storePayloads && ((posCode & 1) != 0))
+ {
+ payloadLen = @in.readVInt();
+ }
+ if ((@in.readVInt() & 1) != 0)
+ {
+ // new offset length
+ @in.readVInt();
+ }
+ if (storePayloads)
+ {
+ @in.skipBytes(payloadLen);
+ }
+ }
+ }
+ }
+
+ if (liveDocs == null || liveDocs.get(accum))
+ {
+ //System.out.println(" return docID=" + accum + " freq=" + freq);
+ return (docID_Renamed = accum);
+ }
+ }
+ }
+
+ public override int docID()
+ {
+ return docID_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int advance(int target) throws java.io.IOException
+ public override int advance(int target)
+ {
+ // TODO: we could make more efficient version, but, it
+ // should be rare that this will matter in practice
+ // since usually apps will not store "big" fields in
+ // this codec!
+ return slowAdvance(target);
+ }
+
+ public override int freq()
+ {
+ return freq_Renamed;
+ }
+
+ public override long cost()
+ {
+ return numDocs;
+ }
+ }
+
+ private sealed class FSTDocsAndPositionsEnum : DocsAndPositionsEnum
+ {
+ internal bool InstanceFieldsInitialized = false;
+
+ internal void InitializeInstanceFields()
+ {
+ @in = new ByteArrayDataInput(buffer);
+ }
+
+ internal readonly bool storePayloads;
+ internal sbyte[] buffer = new sbyte[16];
+ internal ByteArrayDataInput @in;
+
+ internal Bits liveDocs;
+ internal int docUpto;
+ internal int docID_Renamed = -1;
+ internal int accum;
+ internal int freq_Renamed;
+ internal int numDocs;
+ internal int posPending;
+ internal int payloadLength;
+ internal readonly bool storeOffsets;
+ internal int offsetLength;
+ internal int startOffset_Renamed;
+
+ internal int pos;
+ internal readonly BytesRef payload = new BytesRef();
+
+ public FSTDocsAndPositionsEnum(bool storePayloads, bool storeOffsets)
+ {
+ if (!InstanceFieldsInitialized)
+ {
+ InitializeInstanceFields();
+ InstanceFieldsInitialized = true;
+ }
+ this.storePayloads = storePayloads;
+ this.storeOffsets = storeOffsets;
+ }
+
+ public bool canReuse(bool storePayloads, bool storeOffsets)
+ {
+ return storePayloads == this.storePayloads && storeOffsets == this.storeOffsets;
+ }
+
+ public FSTDocsAndPositionsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs)
+ {
+ Debug.Assert(numDocs > 0);
+
+ // System.out.println("D&P reset bytes this=" + this);
+ // for(int i=bufferIn.offset;i<bufferIn.length;i++) {
+ // System.out.println(" " + Integer.toHexString(bufferIn.bytes[i]&0xFF));
+ // }
+
+ if (buffer.Length < bufferIn.length)
+ {
+ buffer = ArrayUtil.grow(buffer, bufferIn.length);
+ }
+ @in.reset(buffer, 0, bufferIn.length - bufferIn.offset);
+ Array.Copy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length);
+ this.liveDocs = liveDocs;
+ docID_Renamed = -1;
+ accum = 0;
+ docUpto = 0;
+ payload.bytes = buffer;
+ payloadLength = 0;
+ this.numDocs = numDocs;
+ posPending = 0;
+ startOffset_Renamed = storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
+ offsetLength = 0;
+ return this;
+ }
+
+ public override int nextDoc()
+ {
+ while (posPending > 0)
+ {
+ nextPosition();
+ }
+ while (true)
+ {
+ //System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
+ if (docUpto == numDocs)
+ {
+ //System.out.println(" END");
+ return docID_Renamed = NO_MORE_DOCS;
+ }
+ docUpto++;
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int code = in.readVInt();
+ int code = @in.readVInt();
+ accum += (int)((uint)code >> 1);
+ if ((code & 1) != 0)
+ {
+ freq_Renamed = 1;
+ }
+ else
+ {
+ freq_Renamed = @in.readVInt();
+ Debug.Assert(freq_Renamed > 0);
+ }
+
+ if (liveDocs == null || liveDocs.get(accum))
+ {
+ pos = 0;
+ startOffset_Renamed = storeOffsets ? 0 : -1;
+ posPending = freq_Renamed;
+ //System.out.println(" return docID=" + accum + " freq=" + freq);
+ return (docID_Renamed = accum);
+ }
+
+ // Skip positions
+ for (int posUpto = 0;posUpto < freq_Renamed;posUpto++)
+ {
+ if (!storePayloads)
+ {
+ @in.readVInt();
+ }
+ else
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int skipCode = in.readVInt();
+ int skipCode = @in.readVInt();
+ if ((skipCode & 1) != 0)
+ {
+ payloadLength = @in.readVInt();
+ //System.out.println(" new payloadLen=" + payloadLength);
+ }
+ }
+
+ if (storeOffsets)
+ {
+ if ((@in.readVInt() & 1) != 0)
+ {
+ // new offset length
+ offsetLength = @in.readVInt();
+ }
+ }
+
+ if (storePayloads)
+ {
+ @in.skipBytes(payloadLength);
+ }
+ }
+ }
+ }
+
+ public override int nextPosition()
+ {
+ //System.out.println(" nextPos storePayloads=" + storePayloads + " this=" + this);
+ Debug.Assert(posPending > 0);
+ posPending--;
+ if (!storePayloads)
+ {
+ pos += @in.readVInt();
+ }
+ else
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int code = in.readVInt();
+ int code = @in.readVInt();
+ pos += (int)((uint)code >> 1);
+ if ((code & 1) != 0)
+ {
+ payloadLength = @in.readVInt();
+ //System.out.println(" new payloadLen=" + payloadLength);
+ //} else {
+ //System.out.println(" same payloadLen=" + payloadLength);
+ }
+ }
+
+ if (storeOffsets)
+ {
+ int offsetCode = @in.readVInt();
+ if ((offsetCode & 1) != 0)
+ {
+ // new offset length
+ offsetLength = @in.readVInt();
+ }
+ startOffset_Renamed += (int)((uint)offsetCode >> 1);
+ }
+
+ if (storePayloads)
+ {
+ payload.offset = @in.Position;
+ @in.skipBytes(payloadLength);
+ payload.length = payloadLength;
+ }
+
+ //System.out.println(" pos=" + pos + " payload=" + payload + " fp=" + in.getPosition());
+ return pos;
+ }
+
+ public override int startOffset()
+ {
+ return startOffset_Renamed;
+ }
+
+ public override int endOffset()
+ {
+ return startOffset_Renamed + offsetLength;
+ }
+
+ public override BytesRef Payload
+ {
+ get
+ {
+ return payload.length > 0 ? payload : null;
+ }
+ }
+
+ public override int docID()
+ {
+ return docID_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int advance(int target) throws java.io.IOException
+ public override int advance(int target)
+ {
+ // TODO: we could make more efficient version, but, it
+ // should be rare that this will matter in practice
+ // since usually apps will not store "big" fields in
+ // this codec!
+ return slowAdvance(target);
+ }
+
+ public override int freq()
+ {
+ return freq_Renamed;
+ }
+
+ public override long cost()
+ {
+ return numDocs;
+ }
+ }
+
+ private sealed class FSTTermsEnum : TermsEnum
+ {
+ internal readonly FieldInfo field;
+ internal readonly BytesRefFSTEnum<BytesRef> fstEnum;
+ internal readonly ByteArrayDataInput buffer = new ByteArrayDataInput();
+ internal bool didDecode;
+
+ internal int docFreq_Renamed;
+ internal long totalTermFreq_Renamed;
+ internal BytesRefFSTEnum.InputOutput<BytesRef> current;
+ internal BytesRef postingsSpare = new BytesRef();
+
+ public FSTTermsEnum(FieldInfo field, FST<BytesRef> fst)
+ {
+ this.field = field;
+ fstEnum = new BytesRefFSTEnum<>(fst);
+ }
+
+ internal void decodeMetaData()
+ {
+ if (!didDecode)
+ {
+ buffer.reset(current.output.bytes, current.output.offset, current.output.length);
+ docFreq_Renamed = buffer.readVInt();
+ if (field.IndexOptions != IndexOptions.DOCS_ONLY)
+ {
+ totalTermFreq_Renamed = docFreq_Renamed + buffer.readVLong();
+ }
+ else
+ {
+ totalTermFreq_Renamed = -1;
+ }
+ postingsSpare.bytes = current.output.bytes;
+ postingsSpare.offset = buffer.Position;
+ postingsSpare.length = current.output.length - (buffer.Position - current.output.offset);
+ //System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length);
+ didDecode = true;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean seekExact(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+ public override bool seekExact(BytesRef text)
+ {
+ //System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
+ current = fstEnum.seekExact(text);
+ didDecode = false;
+ return current != null;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+ public override SeekStatus seekCeil(BytesRef text)
+ {
+ //System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
+ current = fstEnum.seekCeil(text);
+ if (current == null)
+ {
+ return SeekStatus.END;
+ }
+ else
+ {
+
+ // System.out.println(" got term=" + current.input.utf8ToString());
+ // for(int i=0;i<current.output.length;i++) {
+ // System.out.println(" " + Integer.toHexString(current.output.bytes[i]&0xFF));
+ // }
+
+ didDecode = false;
+
+ if (text.Equals(current.input))
+ {
+ //System.out.println(" found!");
+ return SeekStatus.FOUND;
+ }
+ else
+ {
+ //System.out.println(" not found: " + current.input.utf8ToString());
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+ }
+
+ public override DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
+ {
+ decodeMetaData();
+ FSTDocsEnum docsEnum;
+
+ if (reuse == null || !(reuse is FSTDocsEnum))
+ {
+ docsEnum = new FSTDocsEnum(field.IndexOptions, field.hasPayloads());
+ }
+ else
+ {
+ docsEnum = (FSTDocsEnum) reuse;
+ if (!docsEnum.canReuse(field.IndexOptions, field.hasPayloads()))
+ {
+ docsEnum = new FSTDocsEnum(field.IndexOptions, field.hasPayloads());
+ }
+ }
+ return docsEnum.reset(this.postingsSpare, liveDocs, docFreq_Renamed);
+ }
+
+ public override DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags)
+ {
+
+ bool hasOffsets = field.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ if (field.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0)
+ {
+ return null;
+ }
+ decodeMetaData();
+ FSTDocsAndPositionsEnum docsAndPositionsEnum;
+ if (reuse == null || !(reuse is FSTDocsAndPositionsEnum))
+ {
+ docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.hasPayloads(), hasOffsets);
+ }
+ else
+ {
+ docsAndPositionsEnum = (FSTDocsAndPositionsEnum) reuse;
+ if (!docsAndPositionsEnum.canReuse(field.hasPayloads(), hasOffsets))
+ {
+ docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.hasPayloads(), hasOffsets);
+ }
+ }
+ //System.out.println("D&P reset this=" + this);
+ return docsAndPositionsEnum.reset(postingsSpare, liveDocs, docFreq_Renamed);
+ }
+
+ public override BytesRef term()
+ {
+ return current.input;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+ public override BytesRef next()
+ {
+ //System.out.println("te.next");
+ current = fstEnum.next();
+ if (current == null)
+ {
+ //System.out.println(" END");
+ return null;
+ }
+ didDecode = false;
+ //System.out.println(" term=" + field.name + ":" + current.input.utf8ToString());
+ return current.input;
+ }
+
+ public override int docFreq()
+ {
+ decodeMetaData();
+ return docFreq_Renamed;
+ }
+
+ public override long totalTermFreq()
+ {
+ decodeMetaData();
+ return totalTermFreq_Renamed;
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+ public override void seekExact(long ord)
+ {
+ // NOTE: we could add this...
+ throw new System.NotSupportedException();
+ }
+
+ public override long ord()
+ {
+ // NOTE: we could add this...
+ throw new System.NotSupportedException();
+ }
+ }
+
+ private sealed class TermsReader : Terms
+ {
+
+ internal readonly long sumTotalTermFreq;
+ internal readonly long sumDocFreq;
+ internal readonly int docCount;
+ internal readonly int termCount;
+ internal FST<BytesRef> fst;
+ internal readonly ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton;
+ internal readonly FieldInfo field;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public TermsReader(org.apache.lucene.index.FieldInfos fieldInfos, org.apache.lucene.store.IndexInput in, int termCount) throws java.io.IOException
+ public TermsReader(FieldInfos fieldInfos, IndexInput @in, int termCount)
+ {
+ this.termCount = termCount;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int fieldNumber = in.readVInt();
+ int fieldNumber = @in.readVInt();
+ field = fieldInfos.fieldInfo(fieldNumber);
+ if (field.IndexOptions != IndexOptions.DOCS_ONLY)
+ {
+ sumTotalTermFreq = @in.readVLong();
+ }
+ else
+ {
+ sumTotalTermFreq = -1;
+ }
+ sumDocFreq = @in.readVLong();
+ docCount = @in.readVInt();
+
+ fst = new FST<>(@in, outputs);
+ }
+
+ public override long SumTotalTermFreq
+ {
+ get
+ {
+ return sumTotalTermFreq;
+ }
+ }
+
+ public override long SumDocFreq
+ {
+ get
+ {
+ return sumDocFreq;
+ }
+ }
+
+ public override int DocCount
+ {
+ get
+ {
+ return docCount;
+ }
+ }
+
+ public override long size()
+ {
+ return termCount;
+ }
+
+ public override TermsEnum iterator(TermsEnum reuse)
+ {
+ return new FSTTermsEnum(field, fst);
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+ public override bool hasFreqs()
+ {
+ return field.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ }
+
+ public override bool hasOffsets()
+ {
+ return field.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ public override bool hasPositions()
+ {
+ return field.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ public override bool hasPayloads()
+ {
+ return field.hasPayloads();
+ }
+
+ public long ramBytesUsed()
+ {
+ return ((fst != null) ? fst.sizeInBytes() : 0);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override FieldsProducer fieldsProducer(SegmentReadState state)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String fileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+ string fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.store.ChecksumIndexInput in = state.directory.openChecksumInput(fileName, org.apache.lucene.store.IOContext.READONCE);
+ ChecksumIndexInput @in = state.directory.openChecksumInput(fileName, IOContext.READONCE);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.SortedMap<String,TermsReader> fields = new java.util.TreeMap<>();
+ SortedMap<string, TermsReader> fields = new SortedDictionary<string, TermsReader>();
+
+ try
+ {
+ CodecUtil.checkHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
+ while (true)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int termCount = in.readVInt();
+ int termCount = @in.readVInt();
+ if (termCount == 0)
+ {
+ break;
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
+ TermsReader termsReader = new TermsReader(state.fieldInfos, @in, termCount);
+ // System.out.println("load field=" + termsReader.field.name);
+ fields.put(termsReader.field.name, termsReader);
+ }
+ CodecUtil.checkFooter(@in);
+ }
+ finally
+ {
+ @in.close();
+ }
+
+ return new FieldsProducerAnonymousInnerClassHelper(this, fields);
+ }
+
+ private class FieldsProducerAnonymousInnerClassHelper : FieldsProducer
+ {
+ private readonly MemoryPostingsFormat outerInstance;
+
+ private SortedMap<string, TermsReader> fields;
+
+ public FieldsProducerAnonymousInnerClassHelper(MemoryPostingsFormat outerInstance, SortedMap<string, TermsReader> fields)
+ {
+ this.outerInstance = outerInstance;
+ this.fields = fields;
+ }
+
+ public override IEnumerator<string> iterator()
+ {
+ return Collections.unmodifiableSet(fields.Keys).GetEnumerator();
+ }
+
+ public override Terms terms(string field)
+ {
+ return fields.get(field);
+ }
+
+ public override int size()
+ {
+ return fields.size();
+ }
+
+ public override void close()
+ {
+ // Drop ref to FST:
+ foreach (TermsReader termsReader in fields.values())
+ {
+ termsReader.fst = null;
+ }
+ }
+
+ public override long ramBytesUsed()
+ {
+ long sizeInBytes = 0;
+ foreach (KeyValuePair<string, TermsReader> entry in fields.entrySet())
+ {
+ sizeInBytes += (entry.Key.length() * RamUsageEstimator.NUM_BYTES_CHAR);
+ sizeInBytes += entry.Value.ramBytesUsed();
+ }
+ return sizeInBytes;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+ public override void checkIntegrity()
+ {
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/RectangularArrays.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/RectangularArrays.cs b/src/Lucene.Net.Codecs/RectangularArrays.cs
new file mode 100644
index 0000000..9b41c8a
--- /dev/null
+++ b/src/Lucene.Net.Codecs/RectangularArrays.cs
@@ -0,0 +1,29 @@
+//----------------------------------------------------------------------------------------
+// Copyright © 2007 - 2014 Tangible Software Solutions Inc.
+// This class can be used by anyone provided that the copyright notice remains intact.
+//
+// This class provides the logic to simulate Java rectangular arrays, which are jagged
+// arrays with inner arrays of the same length. A size of -1 indicates unknown length.
+//----------------------------------------------------------------------------------------
+internal static partial class RectangularArrays
+{
+ internal static long[][] ReturnRectangularLongArray(int Size1, int Size2)
+ {
+ long[][] Array;
+ if (Size1 > -1)
+ {
+ Array = new long[Size1][];
+ if (Size2 > -1)
+ {
+ for (int Array1 = 0; Array1 < Size1; Array1++)
+ {
+ Array[Array1] = new long[Size2];
+ }
+ }
+ }
+ else
+ Array = null;
+
+ return Array;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/IntIndexInput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/IntIndexInput.cs b/src/Lucene.Net.Codecs/Sep/IntIndexInput.cs
index 057b25b..d08b7d4 100644
--- a/src/Lucene.Net.Codecs/Sep/IntIndexInput.cs
+++ b/src/Lucene.Net.Codecs/Sep/IntIndexInput.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,43 +15,45 @@ package codecs.sep;
* limitations under the License.
*/
-import java.io.Closeable;
-import java.io.IOException;
-
-import store.DataInput;
-
-/** Defines basic API for writing ints to an IndexOutput.
- * IntBlockCodec interacts with this API. @see
- * IntBlockReader
- *
- * @lucene.experimental */
-public abstract class IntIndexInput implements Closeable {
-
- public abstract Reader reader() ;
-
- @Override
- public abstract void close() ;
-
- public abstract Index index() ;
-
- /** Records a single skip-point in the {@link IntIndexInput.Reader}. */
- public abstract static class Index {
-
- public abstract void read(DataInput indexIn, bool absolute) ;
-
- /** Seeks primary stream to the last read offset */
- public abstract void seek(IntIndexInput.Reader stream) ;
-
- public abstract void copyFrom(Index other);
+namespace Lucene.Net.Codecs.Sep
+{
+ using Store;
+ using System;
+
+ /// <summary>
+ /// Defines basic API for writing ints to an IndexOutput.
+ /// IntBlockCodec interacts with this API. @see
+ /// IntBlockReader
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class IntIndexInput : IDisposable
+ {
+ public abstract IntIndexInputReader Reader();
+ public abstract void Dispose();
+ public abstract IntIndexInputIndex Index();
+
+
+ }
- @Override
- public abstract Index clone();
- }
-
- /** Reads int values. */
- public abstract static class Reader {
-
- /** Reads next single int */
- public abstract int next() ;
- }
+ /// <summary>Reads int values</summary>
+ public abstract class IntIndexInputReader
+ {
+ /// <summary>Reads next single int</summary>
+ public abstract int Next();
+ }
+
+ /// <summary>
+ /// Records a single skip-point in the <seealso cref="IntIndexInput.Reader"/>. </summary>
+ public abstract class IntIndexInputIndex
+ {
+ public abstract void Read(DataInput indexIn, bool absolute);
+
+ /// <summary>Seeks primary stream to the last read offset </summary>
+ public abstract void Seek(IntIndexInputReader stream);
+
+ public abstract void CopyFrom(IntIndexInputIndex other);
+
+ public abstract IntIndexInputIndex Clone();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/IntIndexOutput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/IntIndexOutput.cs b/src/Lucene.Net.Codecs/Sep/IntIndexOutput.cs
index b5fe3f9..901dbe1 100644
--- a/src/Lucene.Net.Codecs/Sep/IntIndexOutput.cs
+++ b/src/Lucene.Net.Codecs/Sep/IntIndexOutput.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,45 +15,57 @@ package codecs.sep;
* limitations under the License.
*/
-// TODO: we may want tighter integration w/ IndexOutput --
-// may give better perf:
+namespace Lucene.Net.Codecs.Sep
+{
+ using System;
+ using Store;
-import store.DataOutput;
+ /// <summary>
+ /// Defines basic API for writing ints to an IndexOutput.
+ /// IntBlockCodec interacts with this API. @see IntBlockReader.
+ ///
+ /// NOTE: block sizes could be variable
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ /// <remarks>
+ /// TODO: We may want tighter integration w/IndexOutput
+ /// may give better performance
+ /// </remarks>
+ public abstract class IntIndexOutput : IDisposable
+ {
-import java.io.IOException;
-import java.io.Closeable;
+ /// <summary>
+ /// Write an int to the primary file. The value must be
+ /// >= 0.
+ /// </summary>
+ public abstract void Write(int v);
-/** Defines basic API for writing ints to an IndexOutput.
- * IntBlockCodec interacts with this API. @see
- * IntBlockReader.
- *
- * <p>NOTE: block sizes could be variable
- *
- * @lucene.experimental */
-public abstract class IntIndexOutput implements Closeable {
+ /// <summary>
+ /// If you are indexing the primary output file, call
+ /// this and interact with the returned IndexWriter.
+ /// </summary>
+ public abstract IntIndexOutputIndex Index();
- /** Write an int to the primary file. The value must be
- * >= 0. */
- public abstract void write(int v) ;
+ public abstract void Dispose();
+ }
- /** Records a single skip-point in the IndexOutput. */
- public abstract static class Index {
- /** Internally records the current location */
- public abstract void mark() ;
+ /// <summary>Records a single skip-point in the IndexOutput. </summary>
+ public abstract class IntIndexOutputIndex
+ {
- /** Copies index from other */
- public abstract void copyFrom(Index other, bool copyLast) ;
+ /// <summary>Internally records the current location </summary>
+ public abstract void Mark();
- /** Writes "location" of current output pointer of primary
- * output to different output (out) */
- public abstract void write(DataOutput indexOut, bool absolute) ;
- }
+ /// <summary>Copies index from other </summary>
+ public abstract void CopyFrom(IntIndexOutputIndex other, bool copyLast);
- /** If you are indexing the primary output file, call
- * this and interact with the returned IndexWriter. */
- public abstract Index index();
+ /// <summary>
+ /// Writes "location" of current output pointer of primary
+ /// output to different output (out)
+ /// </summary>
+ public abstract void Write(DataOutput indexOut, bool absolute);
+ }
- @Override
- public abstract void close() ;
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Sep/IntStreamFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Sep/IntStreamFactory.cs b/src/Lucene.Net.Codecs/Sep/IntStreamFactory.cs
index 8e23a76..a718d27 100644
--- a/src/Lucene.Net.Codecs/Sep/IntStreamFactory.cs
+++ b/src/Lucene.Net.Codecs/Sep/IntStreamFactory.cs
@@ -1,6 +1,4 @@
-package codecs.sep;
-
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,20 +15,28 @@ package codecs.sep;
* limitations under the License.
*/
-import store.Directory;
-import store.IOContext;
+namespace Lucene.Net.Codecs.Sep
+{
-import java.io.IOException;
+ using Store;
-/** Provides int reader and writer to specified files.
- *
- * @lucene.experimental */
-public abstract class IntStreamFactory {
- /** Create an {@link IntIndexInput} on the provided
- * fileName. */
- public abstract IntIndexInput openInput(Directory dir, String fileName, IOContext context) ;
+ /// <summary>
+ /// Provides int reader and writer to specified files.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class IntStreamFactory
+ {
+
+ /// <summary>
+ /// Create an <seealso cref="IntIndexInput"/> on the provided fileName.
+ /// </summary>
+ public abstract IntIndexInput OpenInput(Directory dir, string fileName, IOContext context);
+
+ /// <summary>
+ /// Create an <seealso cref="IntIndexOutput"/> on the provided fileName.
+ /// </summary>
+ public abstract IntIndexOutput CreateOutput(Directory dir, string fileName, IOContext context);
+ }
- /** Create an {@link IntIndexOutput} on the provided
- * fileName. */
- public abstract IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) ;
-}
+}
\ No newline at end of file
[08/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTOrdPostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdPostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdPostingsFormat.cs
index 0545afc..5723f48 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTOrdPostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTOrdPostingsFormat.cs
@@ -1,83 +1,88 @@
-package codecs.memory;
+namespace org.apache.lucene.codecs.memory
+{
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.io.IOException;
+ using Lucene41PostingsWriter = org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
+ using Lucene41PostingsReader = org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using IOUtils = org.apache.lucene.util.IOUtils;
-import codecs.FieldsConsumer;
-import codecs.FieldsProducer;
-import codecs.PostingsFormat;
-import codecs.PostingsReaderBase;
-import codecs.PostingsWriterBase;
-import codecs.lucene41.Lucene41PostingsWriter;
-import codecs.lucene41.Lucene41PostingsReader;
-import index.FieldInfo.IndexOptions;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import util.IOUtils;
+ /// <summary>
+ /// FSTOrd term dict + Lucene41PBF
+ /// </summary>
-/**
- * FSTOrd term dict + Lucene41PBF
- */
+ public sealed class FSTOrdPostingsFormat : PostingsFormat
+ {
+ public FSTOrdPostingsFormat() : base("FSTOrd41")
+ {
+ }
-public final class FSTOrdPostingsFormat extends PostingsFormat {
- public FSTOrdPostingsFormat() {
- super("FSTOrd41");
- }
+ public override string ToString()
+ {
+ return Name;
+ }
- @Override
- public String toString() {
- return getName();
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
- PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);
+ bool success = false;
+ try
+ {
+ FieldsConsumer ret = new FSTOrdTermsWriter(state, postingsWriter);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(postingsWriter);
+ }
+ }
+ }
- bool success = false;
- try {
- FieldsConsumer ret = new FSTOrdTermsWriter(state, postingsWriter);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override FieldsProducer fieldsProducer(SegmentReadState state)
+ {
+ PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ bool success = false;
+ try
+ {
+ FieldsProducer ret = new FSTOrdTermsReader(state, postingsReader);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(postingsReader);
+ }
+ }
+ }
+ }
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) {
- PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- state.context,
- state.segmentSuffix);
- bool success = false;
- try {
- FieldsProducer ret = new FSTOrdTermsReader(state, postingsReader);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsReader);
- }
- }
- }
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTOrdPulsing41PostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdPulsing41PostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdPulsing41PostingsFormat.cs
index acb5645..8d0317d 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTOrdPulsing41PostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTOrdPulsing41PostingsFormat.cs
@@ -1,91 +1,101 @@
-package codecs.memory;
+namespace org.apache.lucene.codecs.memory
+{
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.io.IOException;
+ using Lucene41PostingsWriter = org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
+ using Lucene41PostingsReader = org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+ using Lucene41PostingsBaseFormat = org.apache.lucene.codecs.lucene41.Lucene41PostingsBaseFormat;
+ using Lucene41PostingsFormat = org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
+ using PulsingPostingsWriter = org.apache.lucene.codecs.pulsing.PulsingPostingsWriter;
+ using PulsingPostingsReader = org.apache.lucene.codecs.pulsing.PulsingPostingsReader;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using IOUtils = org.apache.lucene.util.IOUtils;
-import codecs.FieldsConsumer;
-import codecs.FieldsProducer;
-import codecs.PostingsBaseFormat;
-import codecs.PostingsFormat;
-import codecs.PostingsReaderBase;
-import codecs.PostingsWriterBase;
-import codecs.lucene41.Lucene41PostingsWriter;
-import codecs.lucene41.Lucene41PostingsReader;
-import codecs.lucene41.Lucene41PostingsBaseFormat;
-import codecs.lucene41.Lucene41PostingsFormat;
-import codecs.pulsing.PulsingPostingsWriter;
-import codecs.pulsing.PulsingPostingsReader;
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import util.IOUtils;
+ /// <summary>
+ /// FSTOrd + Pulsing41
+ /// @lucene.experimental
+ /// </summary>
-/** FSTOrd + Pulsing41
- * @lucene.experimental */
+ public class FSTOrdPulsing41PostingsFormat : PostingsFormat
+ {
+ private readonly PostingsBaseFormat wrappedPostingsBaseFormat;
+ private readonly int freqCutoff;
-public class FSTOrdPulsing41PostingsFormat extends PostingsFormat {
- private final PostingsBaseFormat wrappedPostingsBaseFormat;
- private final int freqCutoff;
+ public FSTOrdPulsing41PostingsFormat() : this(1)
+ {
+ }
- public FSTOrdPulsing41PostingsFormat() {
- this(1);
- }
-
- public FSTOrdPulsing41PostingsFormat(int freqCutoff) {
- super("FSTOrdPulsing41");
- this.wrappedPostingsBaseFormat = new Lucene41PostingsBaseFormat();
- this.freqCutoff = freqCutoff;
- }
+ public FSTOrdPulsing41PostingsFormat(int freqCutoff) : base("FSTOrdPulsing41")
+ {
+ this.wrappedPostingsBaseFormat = new Lucene41PostingsBaseFormat();
+ this.freqCutoff = freqCutoff;
+ }
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
- PostingsWriterBase docsWriter = null;
- PostingsWriterBase pulsingWriter = null;
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ PostingsWriterBase docsWriter = null;
+ PostingsWriterBase pulsingWriter = null;
- bool success = false;
- try {
- docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state);
- pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter);
- FieldsConsumer ret = new FSTOrdTermsWriter(state, pulsingWriter);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter);
- }
- }
- }
+ bool success = false;
+ try
+ {
+ docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state);
+ pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter);
+ FieldsConsumer ret = new FSTOrdTermsWriter(state, pulsingWriter);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter);
+ }
+ }
+ }
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) {
- PostingsReaderBase docsReader = null;
- PostingsReaderBase pulsingReader = null;
- bool success = false;
- try {
- docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state);
- pulsingReader = new PulsingPostingsReader(state, docsReader);
- FieldsProducer ret = new FSTOrdTermsReader(state, pulsingReader);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docsReader, pulsingReader);
- }
- }
- }
-}
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.FieldsProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override FieldsProducer fieldsProducer(SegmentReadState state)
+ {
+ PostingsReaderBase docsReader = null;
+ PostingsReaderBase pulsingReader = null;
+ bool success = false;
+ try
+ {
+ docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state);
+ pulsingReader = new PulsingPostingsReader(state, docsReader);
+ FieldsProducer ret = new FSTOrdTermsReader(state, pulsingReader);
+ success = true;
+ return ret;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(docsReader, pulsingReader);
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
index 53d0d69..f57f231 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
@@ -1,860 +1,1154 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.TreeMap;
-
-import codecs.BlockTermState;
-import codecs.CodecUtil;
-import codecs.FieldsProducer;
-import codecs.PostingsReaderBase;
-import codecs.memory.FSTTermsReader.TermsReader;
-import index.CorruptIndexException;
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.SegmentInfo;
-import index.SegmentReadState;
-import index.TermState;
-import index.Terms;
-import index.TermsEnum;
-import store.ByteArrayDataInput;
-import store.ChecksumIndexInput;
-import store.IndexInput;
-import util.ArrayUtil;
-import util.Bits;
-import util.BytesRef;
-import util.IOUtils;
-import util.RamUsageEstimator;
-import util.automaton.ByteRunAutomaton;
-import util.automaton.CompiledAutomaton;
-import util.fst.BytesRefFSTEnum.InputOutput;
-import util.fst.BytesRefFSTEnum;
-import util.fst.FST;
-import util.fst.Outputs;
-import util.fst.PositiveIntOutputs;
-import util.fst.Util;
-
-/**
- * FST-based terms dictionary reader.
- *
- * The FST index maps each term and its ord, and during seek
- * the ord is used fetch metadata from a single block.
- * The term dictionary is fully memory resident.
- *
- * @lucene.experimental
- */
-public class FSTOrdTermsReader extends FieldsProducer {
- static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
- final TreeMap<String, TermsReader> fields = new TreeMap<>();
- final PostingsReaderBase postingsReader;
- int version;
- //static final bool TEST = false;
-
- public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) {
- final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_INDEX_EXTENSION);
- final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);
-
- this.postingsReader = postingsReader;
- ChecksumIndexInput indexIn = null;
- IndexInput blockIn = null;
- bool success = false;
- try {
- indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
- blockIn = state.directory.openInput(termsBlockFileName, state.context);
- version = readHeader(indexIn);
- readHeader(blockIn);
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(blockIn);
- }
-
- this.postingsReader.init(blockIn);
- seekDir(blockIn);
-
- final FieldInfos fieldInfos = state.fieldInfos;
- final int numFields = blockIn.readVInt();
- for (int i = 0; i < numFields; i++) {
- FieldInfo fieldInfo = fieldInfos.fieldInfo(blockIn.readVInt());
- bool hasFreq = fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY;
- long numTerms = blockIn.readVLong();
- long sumTotalTermFreq = hasFreq ? blockIn.readVLong() : -1;
- long sumDocFreq = blockIn.readVLong();
- int docCount = blockIn.readVInt();
- int longsSize = blockIn.readVInt();
- FST<Long> index = new FST<>(indexIn, PositiveIntOutputs.getSingleton());
-
- TermsReader current = new TermsReader(fieldInfo, blockIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
- TermsReader previous = fields.put(fieldInfo.name, current);
- checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
- }
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checkFooter(indexIn);
- } else {
- CodecUtil.checkEOF(indexIn);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(indexIn, blockIn);
- } else {
- IOUtils.closeWhileHandlingException(indexIn, blockIn);
- }
- }
- }
-
- private int readHeader(IndexInput in) {
- return CodecUtil.checkHeader(in, FSTOrdTermsWriter.TERMS_CODEC_NAME,
- FSTOrdTermsWriter.TERMS_VERSION_START,
- FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
- }
- private void seekDir(IndexInput in) {
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- in.seek(in.length() - CodecUtil.footerLength() - 8);
- } else {
- in.seek(in.length() - 8);
- }
- in.seek(in.readLong());
- }
- private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) {
- // #docs with field must be <= #docs
- if (field.docCount < 0 || field.docCount > info.getDocCount()) {
- throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + indexIn + ", " + blockIn + ")");
- }
- // #postings must be >= #docs with field
- if (field.sumDocFreq < field.docCount) {
- throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + indexIn + ", " + blockIn + ")");
- }
- // #positions must be >= #postings
- if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) {
- throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + indexIn + ", " + blockIn + ")");
- }
- if (previous != null) {
- throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + indexIn + ", " + blockIn + ")");
- }
- }
-
- @Override
- public Iterator<String> iterator() {
- return Collections.unmodifiableSet(fields.keySet()).iterator();
- }
-
- @Override
- public Terms terms(String field) {
- Debug.Assert( field != null;
- return fields.get(field);
- }
-
- @Override
- public int size() {
- return fields.size();
- }
-
- @Override
- public void close() {
- try {
- IOUtils.close(postingsReader);
- } finally {
- fields.clear();
- }
- }
-
- final class TermsReader extends Terms {
- final FieldInfo fieldInfo;
- final long numTerms;
- final long sumTotalTermFreq;
- final long sumDocFreq;
- final int docCount;
- final int longsSize;
- final FST<Long> index;
-
- final int numSkipInfo;
- final long[] skipInfo;
- final byte[] statsBlock;
- final byte[] metaLongsBlock;
- final byte[] metaBytesBlock;
-
- TermsReader(FieldInfo fieldInfo, IndexInput blockIn, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST<Long> index) {
- this.fieldInfo = fieldInfo;
- this.numTerms = numTerms;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
- this.longsSize = longsSize;
- this.index = index;
-
- Debug.Assert( (numTerms & (~0xffffffffL)) == 0;
- final int numBlocks = (int)(numTerms + INTERVAL - 1) / INTERVAL;
- this.numSkipInfo = longsSize + 3;
- this.skipInfo = new long[numBlocks * numSkipInfo];
- this.statsBlock = new byte[(int)blockIn.readVLong()];
- this.metaLongsBlock = new byte[(int)blockIn.readVLong()];
- this.metaBytesBlock = new byte[(int)blockIn.readVLong()];
-
- int last = 0, next = 0;
- for (int i = 1; i < numBlocks; i++) {
- next = numSkipInfo * i;
- for (int j = 0; j < numSkipInfo; j++) {
- skipInfo[next + j] = skipInfo[last + j] + blockIn.readVLong();
- }
- last = next;
- }
- blockIn.readBytes(statsBlock, 0, statsBlock.length);
- blockIn.readBytes(metaLongsBlock, 0, metaLongsBlock.length);
- blockIn.readBytes(metaBytesBlock, 0, metaBytesBlock.length);
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public bool hasFreqs() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- }
-
- @Override
- public bool hasOffsets() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
-
- @Override
- public bool hasPositions() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
-
- @Override
- public bool hasPayloads() {
- return fieldInfo.hasPayloads();
- }
-
- @Override
- public long size() {
- return numTerms;
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return sumTotalTermFreq;
- }
-
- @Override
- public long getSumDocFreq() {
- return sumDocFreq;
- }
-
- @Override
- public int getDocCount() {
- return docCount;
- }
-
- @Override
- public TermsEnum iterator(TermsEnum reuse) {
- return new SegmentTermsEnum();
- }
-
- @Override
- public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) {
- return new IntersectTermsEnum(compiled, startTerm);
- }
-
- // Only wraps common operations for PBF interact
- abstract class BaseTermsEnum extends TermsEnum {
- /* Current term, null when enum ends or unpositioned */
- BytesRef term;
-
- /* Current term's ord, starts from 0 */
- long ord;
-
- /* Current term stats + decoded metadata (customized by PBF) */
- final BlockTermState state;
-
- /* Datainput to load stats & metadata */
- final ByteArrayDataInput statsReader = new ByteArrayDataInput();
- final ByteArrayDataInput metaLongsReader = new ByteArrayDataInput();
- final ByteArrayDataInput metaBytesReader = new ByteArrayDataInput();
-
- /* To which block is buffered */
- int statsBlockOrd;
- int metaBlockOrd;
-
- /* Current buffered metadata (long[] & byte[]) */
- long[][] longs;
- int[] bytesStart;
- int[] bytesLength;
-
- /* Current buffered stats (df & ttf) */
- int[] docFreq;
- long[] totalTermFreq;
-
- BaseTermsEnum() {
- this.state = postingsReader.newTermState();
- this.term = null;
- this.statsReader.reset(statsBlock);
- this.metaLongsReader.reset(metaLongsBlock);
- this.metaBytesReader.reset(metaBytesBlock);
-
- this.longs = new long[INTERVAL][longsSize];
- this.bytesStart = new int[INTERVAL];
- this.bytesLength = new int[INTERVAL];
- this.docFreq = new int[INTERVAL];
- this.totalTermFreq = new long[INTERVAL];
- this.statsBlockOrd = -1;
- this.metaBlockOrd = -1;
- if (!hasFreqs()) {
- Arrays.fill(totalTermFreq, -1);
- }
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- /** Decodes stats data into term state */
- void decodeStats() {
- final int upto = (int)ord % INTERVAL;
- final int oldBlockOrd = statsBlockOrd;
- statsBlockOrd = (int)ord / INTERVAL;
- if (oldBlockOrd != statsBlockOrd) {
- refillStats();
- }
- state.docFreq = docFreq[upto];
- state.totalTermFreq = totalTermFreq[upto];
- }
-
- /** Let PBF decode metadata */
- void decodeMetaData() {
- final int upto = (int)ord % INTERVAL;
- final int oldBlockOrd = metaBlockOrd;
- metaBlockOrd = (int)ord / INTERVAL;
- if (metaBlockOrd != oldBlockOrd) {
- refillMetadata();
- }
- metaBytesReader.setPosition(bytesStart[upto]);
- postingsReader.decodeTerm(longs[upto], metaBytesReader, fieldInfo, state, true);
- }
-
- /** Load current stats shard */
- final void refillStats() {
- final int offset = statsBlockOrd * numSkipInfo;
- final int statsFP = (int)skipInfo[offset];
- statsReader.setPosition(statsFP);
- for (int i = 0; i < INTERVAL && !statsReader.eof(); i++) {
- int code = statsReader.readVInt();
- if (hasFreqs()) {
- docFreq[i] = (code >>> 1);
- if ((code & 1) == 1) {
- totalTermFreq[i] = docFreq[i];
- } else {
- totalTermFreq[i] = docFreq[i] + statsReader.readVLong();
- }
- } else {
- docFreq[i] = code;
- }
- }
- }
-
- /** Load current metadata shard */
- final void refillMetadata() {
- final int offset = metaBlockOrd * numSkipInfo;
- final int metaLongsFP = (int)skipInfo[offset + 1];
- final int metaBytesFP = (int)skipInfo[offset + 2];
- metaLongsReader.setPosition(metaLongsFP);
- for (int j = 0; j < longsSize; j++) {
- longs[0][j] = skipInfo[offset + 3 + j] + metaLongsReader.readVLong();
- }
- bytesStart[0] = metaBytesFP;
- bytesLength[0] = (int)metaLongsReader.readVLong();
- for (int i = 1; i < INTERVAL && !metaLongsReader.eof(); i++) {
- for (int j = 0; j < longsSize; j++) {
- longs[i][j] = longs[i-1][j] + metaLongsReader.readVLong();
- }
- bytesStart[i] = bytesStart[i-1] + bytesLength[i-1];
- bytesLength[i] = (int)metaLongsReader.readVLong();
- }
- }
-
- @Override
- public TermState termState() {
- decodeMetaData();
- return state.clone();
- }
-
- @Override
- public BytesRef term() {
- return term;
- }
-
- @Override
- public int docFreq() {
- return state.docFreq;
- }
-
- @Override
- public long totalTermFreq() {
- return state.totalTermFreq;
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
- decodeMetaData();
- return postingsReader.docs(fieldInfo, state, liveDocs, reuse, flags);
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
- if (!hasPositions()) {
- return null;
- }
- decodeMetaData();
- return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags);
- }
-
- // TODO: this can be achieved by making use of Util.getByOutput()
- // and should have related tests
- @Override
- public void seekExact(long ord) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long ord() {
- throw new UnsupportedOperationException();
- }
- }
-
- // Iterates through all terms in this field
- private final class SegmentTermsEnum extends BaseTermsEnum {
- final BytesRefFSTEnum<Long> fstEnum;
-
- /* True when current term's metadata is decoded */
- bool decoded;
-
- /* True when current enum is 'positioned' by seekExact(TermState) */
- bool seekPending;
-
- SegmentTermsEnum() {
- this.fstEnum = new BytesRefFSTEnum<>(index);
- this.decoded = false;
- this.seekPending = false;
- }
-
- @Override
- void decodeMetaData() {
- if (!decoded && !seekPending) {
- super.decodeMetaData();
- decoded = true;
- }
- }
-
- // Update current enum according to FSTEnum
- void updateEnum(final InputOutput<Long> pair) {
- if (pair == null) {
- term = null;
- } else {
- term = pair.input;
- ord = pair.output;
- decodeStats();
- }
- decoded = false;
- seekPending = false;
- }
-
- @Override
- public BytesRef next() {
- if (seekPending) { // previously positioned, but termOutputs not fetched
- seekPending = false;
- SeekStatus status = seekCeil(term);
- Debug.Assert( status == SeekStatus.FOUND; // must positioned on valid term
- }
- updateEnum(fstEnum.next());
- return term;
- }
-
- @Override
- public bool seekExact(BytesRef target) {
- updateEnum(fstEnum.seekExact(target));
- return term != null;
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef target) {
- updateEnum(fstEnum.seekCeil(target));
- if (term == null) {
- return SeekStatus.END;
- } else {
- return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
- }
- }
-
- @Override
- public void seekExact(BytesRef target, TermState otherState) {
- if (!target.equals(term)) {
- state.copyFrom(otherState);
- term = BytesRef.deepCopyOf(target);
- seekPending = true;
- }
- }
- }
-
- // Iterates intersect result with automaton (cannot seek!)
- private final class IntersectTermsEnum extends BaseTermsEnum {
- /* True when current term's metadata is decoded */
- bool decoded;
-
- /* True when there is pending term when calling next() */
- bool pending;
-
- /* stack to record how current term is constructed,
- * used to accumulate metadata or rewind term:
- * level == term.length + 1,
- * == 0 when term is null */
- Frame[] stack;
- int level;
-
- /* term dict fst */
- final FST<Long> fst;
- final FST.BytesReader fstReader;
- final Outputs<Long> fstOutputs;
-
- /* query automaton to intersect with */
- final ByteRunAutomaton fsa;
-
- private final class Frame {
- /* fst stats */
- FST.Arc<Long> arc;
-
- /* automaton stats */
- int state;
-
- Frame() {
- this.arc = new FST.Arc<>();
- this.state = -1;
- }
-
- public String toString() {
- return "arc=" + arc + " state=" + state;
- }
- }
-
- IntersectTermsEnum(CompiledAutomaton compiled, BytesRef startTerm) {
- //if (TEST) System.out.println("Enum init, startTerm=" + startTerm);
- this.fst = index;
- this.fstReader = fst.getBytesReader();
- this.fstOutputs = index.outputs;
- this.fsa = compiled.runAutomaton;
- this.level = -1;
- this.stack = new Frame[16];
- for (int i = 0 ; i < stack.length; i++) {
- this.stack[i] = new Frame();
- }
-
- Frame frame;
- frame = loadVirtualFrame(newFrame());
- this.level++;
- frame = loadFirstFrame(newFrame());
- pushFrame(frame);
-
- this.decoded = false;
- this.pending = false;
-
- if (startTerm == null) {
- pending = isAccept(topFrame());
- } else {
- doSeekCeil(startTerm);
- pending = !startTerm.equals(term) && isValid(topFrame()) && isAccept(topFrame());
- }
- }
-
- @Override
- void decodeMetaData() {
- if (!decoded) {
- super.decodeMetaData();
- decoded = true;
- }
- }
-
- @Override
- void decodeStats() {
- final FST.Arc<Long> arc = topFrame().arc;
- Debug.Assert( arc.nextFinalOutput == fstOutputs.getNoOutput();
- ord = arc.output;
- super.decodeStats();
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef target) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BytesRef next() {
- //if (TEST) System.out.println("Enum next()");
- if (pending) {
- pending = false;
- decodeStats();
- return term;
- }
- decoded = false;
- DFS:
- while (level > 0) {
- Frame frame = newFrame();
- if (loadExpandFrame(topFrame(), frame) != null) { // has valid target
- pushFrame(frame);
- if (isAccept(frame)) { // gotcha
- break;
- }
- continue; // check next target
- }
- frame = popFrame();
- while(level > 0) {
- if (loadNextFrame(topFrame(), frame) != null) { // has valid sibling
- pushFrame(frame);
- if (isAccept(frame)) { // gotcha
- break DFS;
- }
- continue DFS; // check next target
- }
- frame = popFrame();
- }
- return null;
- }
- decodeStats();
- return term;
- }
-
- BytesRef doSeekCeil(BytesRef target) {
- //if (TEST) System.out.println("Enum doSeekCeil()");
- Frame frame= null;
- int label, upto = 0, limit = target.length;
- while (upto < limit) { // to target prefix, or ceil label (rewind prefix)
- frame = newFrame();
- label = target.bytes[upto] & 0xff;
- frame = loadCeilFrame(label, topFrame(), frame);
- if (frame == null || frame.arc.label != label) {
- break;
- }
- Debug.Assert( isValid(frame); // target must be fetched from automaton
- pushFrame(frame);
- upto++;
- }
- if (upto == limit) { // got target
- return term;
- }
- if (frame != null) { // got larger term('s prefix)
- pushFrame(frame);
- return isAccept(frame) ? term : next();
- }
- while (level > 0) { // got target's prefix, advance to larger term
- frame = popFrame();
- while (level > 0 && !canRewind(frame)) {
- frame = popFrame();
- }
- if (loadNextFrame(topFrame(), frame) != null) {
- pushFrame(frame);
- return isAccept(frame) ? term : next();
- }
- }
- return null;
- }
-
- /** Virtual frame, never pop */
- Frame loadVirtualFrame(Frame frame) {
- frame.arc.output = fstOutputs.getNoOutput();
- frame.arc.nextFinalOutput = fstOutputs.getNoOutput();
- frame.state = -1;
- return frame;
- }
-
- /** Load frame for start arc(node) on fst */
- Frame loadFirstFrame(Frame frame) {
- frame.arc = fst.getFirstArc(frame.arc);
- frame.state = fsa.getInitialState();
- return frame;
- }
-
- /** Load frame for target arc(node) on fst */
- Frame loadExpandFrame(Frame top, Frame frame) {
- if (!canGrow(top)) {
- return null;
- }
- frame.arc = fst.readFirstRealTargetArc(top.arc.target, frame.arc, fstReader);
- frame.state = fsa.step(top.state, frame.arc.label);
- //if (TEST) System.out.println(" loadExpand frame="+frame);
- if (frame.state == -1) {
- return loadNextFrame(top, frame);
- }
- return frame;
- }
-
- /** Load frame for sibling arc(node) on fst */
- Frame loadNextFrame(Frame top, Frame frame) {
- if (!canRewind(frame)) {
- return null;
- }
- while (!frame.arc.isLast()) {
- frame.arc = fst.readNextRealArc(frame.arc, fstReader);
- frame.state = fsa.step(top.state, frame.arc.label);
- if (frame.state != -1) {
- break;
- }
- }
- //if (TEST) System.out.println(" loadNext frame="+frame);
- if (frame.state == -1) {
- return null;
- }
- return frame;
- }
-
- /** Load frame for target arc(node) on fst, so that
- * arc.label >= label and !fsa.reject(arc.label) */
- Frame loadCeilFrame(int label, Frame top, Frame frame) {
- FST.Arc<Long> arc = frame.arc;
- arc = Util.readCeilArc(label, fst, top.arc, arc, fstReader);
- if (arc == null) {
- return null;
- }
- frame.state = fsa.step(top.state, arc.label);
- //if (TEST) System.out.println(" loadCeil frame="+frame);
- if (frame.state == -1) {
- return loadNextFrame(top, frame);
- }
- return frame;
- }
-
- bool isAccept(Frame frame) { // reach a term both fst&fsa accepts
- return fsa.isAccept(frame.state) && frame.arc.isFinal();
- }
- bool isValid(Frame frame) { // reach a prefix both fst&fsa won't reject
- return /*frame != null &&*/ frame.state != -1;
- }
- bool canGrow(Frame frame) { // can walk forward on both fst&fsa
- return frame.state != -1 && FST.targetHasArcs(frame.arc);
- }
- bool canRewind(Frame frame) { // can jump to sibling
- return !frame.arc.isLast();
- }
-
- void pushFrame(Frame frame) {
- final FST.Arc<Long> arc = frame.arc;
- arc.output = fstOutputs.add(topFrame().arc.output, arc.output);
- term = grow(arc.label);
- level++;
- Debug.Assert( frame == stack[level];
- }
-
- Frame popFrame() {
- term = shrink();
- return stack[level--];
- }
-
- Frame newFrame() {
- if (level+1 == stack.length) {
- final Frame[] temp = new Frame[ArrayUtil.oversize(level+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(stack, 0, temp, 0, stack.length);
- for (int i = stack.length; i < temp.length; i++) {
- temp[i] = new Frame();
- }
- stack = temp;
- }
- return stack[level+1];
- }
-
- Frame topFrame() {
- return stack[level];
- }
-
- BytesRef grow(int label) {
- if (term == null) {
- term = new BytesRef(new byte[16], 0, 0);
- } else {
- if (term.length == term.bytes.length) {
- term.grow(term.length+1);
- }
- term.bytes[term.length++] = (byte)label;
- }
- return term;
- }
-
- BytesRef shrink() {
- if (term.length == 0) {
- term = null;
- } else {
- term.length--;
- }
- return term;
- }
- }
- }
-
- static<T> void walk(FST<T> fst) {
- final ArrayList<FST.Arc<T>> queue = new ArrayList<>();
- final BitSet seen = new BitSet();
- final FST.BytesReader reader = fst.getBytesReader();
- final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());
- queue.add(startArc);
- while (!queue.isEmpty()) {
- final FST.Arc<T> arc = queue.remove(0);
- final long node = arc.target;
- //System.out.println(arc);
- if (FST.targetHasArcs(arc) && !seen.get((int) node)) {
- seen.set((int) node);
- fst.readFirstRealTargetArc(node, arc, reader);
- while (true) {
- queue.add(new FST.Arc<T>().copyFrom(arc));
- if (arc.isLast()) {
- break;
- } else {
- fst.readNextRealArc(arc, reader);
- }
- }
- }
- }
- }
-
- @Override
- public long ramBytesUsed() {
- long ramBytesUsed = 0;
- for (TermsReader r : fields.values()) {
- if (r.index != null) {
- ramBytesUsed += r.index.sizeInBytes();
- ramBytesUsed += RamUsageEstimator.sizeOf(r.metaBytesBlock);
- ramBytesUsed += RamUsageEstimator.sizeOf(r.metaLongsBlock);
- ramBytesUsed += RamUsageEstimator.sizeOf(r.skipInfo);
- ramBytesUsed += RamUsageEstimator.sizeOf(r.statsBlock);
- }
- }
- return ramBytesUsed;
- }
-
- @Override
- public void checkIntegrity() {
- postingsReader.checkIntegrity();
- }
-}
+using System;
+using System.Diagnostics;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using TermsReader = org.apache.lucene.codecs.memory.FSTTermsReader.TermsReader;
+ using CorruptIndexException = org.apache.lucene.index.CorruptIndexException;
+ using DocsAndPositionsEnum = org.apache.lucene.index.DocsAndPositionsEnum;
+ using DocsEnum = org.apache.lucene.index.DocsEnum;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using FieldInfos = org.apache.lucene.index.FieldInfos;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentInfo = org.apache.lucene.index.SegmentInfo;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using TermState = org.apache.lucene.index.TermState;
+ using Terms = org.apache.lucene.index.Terms;
+ using TermsEnum = org.apache.lucene.index.TermsEnum;
+ using ByteArrayDataInput = org.apache.lucene.store.ByteArrayDataInput;
+ using ChecksumIndexInput = org.apache.lucene.store.ChecksumIndexInput;
+ using IndexInput = org.apache.lucene.store.IndexInput;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using Bits = org.apache.lucene.util.Bits;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using RamUsageEstimator = org.apache.lucene.util.RamUsageEstimator;
+ using ByteRunAutomaton = org.apache.lucene.util.automaton.ByteRunAutomaton;
+ using CompiledAutomaton = org.apache.lucene.util.automaton.CompiledAutomaton;
+ using InputOutput = org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
+ using BytesRefFSTEnum = org.apache.lucene.util.fst.BytesRefFSTEnum;
+ using FST = org.apache.lucene.util.fst.FST;
+ using Outputs = org.apache.lucene.util.fst.Outputs;
+ using PositiveIntOutputs = org.apache.lucene.util.fst.PositiveIntOutputs;
+ using Util = org.apache.lucene.util.fst.Util;
+
+ /// <summary>
+ /// FST-based terms dictionary reader.
+ ///
+ /// The FST index maps each term and its ord, and during seek
+ /// the ord is used fetch metadata from a single block.
+ /// The term dictionary is fully memory resident.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public class FSTOrdTermsReader : FieldsProducer
+ {
+ internal const int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
+ internal readonly SortedDictionary<string, TermsReader> fields = new SortedDictionary<string, TermsReader>();
+ internal readonly PostingsReaderBase postingsReader;
+ internal int version;
+ //static final boolean TEST = false;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public FSTOrdTermsReader(org.apache.lucene.index.SegmentReadState state, org.apache.lucene.codecs.PostingsReaderBase postingsReader) throws java.io.IOException
+ public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String termsIndexFileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_INDEX_EXTENSION);
+ string termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_INDEX_EXTENSION);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String termsBlockFileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);
+ string termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);
+
+ this.postingsReader = postingsReader;
+ ChecksumIndexInput indexIn = null;
+ IndexInput blockIn = null;
+ bool success = false;
+ try
+ {
+ indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
+ blockIn = state.directory.openInput(termsBlockFileName, state.context);
+ version = readHeader(indexIn);
+ readHeader(blockIn);
+ if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
+ {
+ CodecUtil.checksumEntireFile(blockIn);
+ }
+
+ this.postingsReader.init(blockIn);
+ seekDir(blockIn);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.FieldInfos fieldInfos = state.fieldInfos;
+ FieldInfos fieldInfos = state.fieldInfos;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int numFields = blockIn.readVInt();
+ int numFields = blockIn.readVInt();
+ for (int i = 0; i < numFields; i++)
+ {
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(blockIn.readVInt());
+ bool hasFreq = fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY;
+ long numTerms = blockIn.readVLong();
+ long sumTotalTermFreq = hasFreq ? blockIn.readVLong() : -1;
+ long sumDocFreq = blockIn.readVLong();
+ int docCount = blockIn.readVInt();
+ int longsSize = blockIn.readVInt();
+ FST<long?> index = new FST<long?>(indexIn, PositiveIntOutputs.Singleton);
+
+ TermsReader current = new TermsReader(fieldInfo, blockIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
+ TermsReader previous = fields[fieldInfo.name] = current;
+ checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
+ }
+ if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
+ {
+ CodecUtil.checkFooter(indexIn);
+ }
+ else
+ {
+ CodecUtil.checkEOF(indexIn);
+ }
+ success = true;
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.close(indexIn, blockIn);
+ }
+ else
+ {
+ IOUtils.closeWhileHandlingException(indexIn, blockIn);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private int readHeader(org.apache.lucene.store.IndexInput in) throws java.io.IOException
+ private int readHeader(IndexInput @in)
+ {
+ return CodecUtil.checkHeader(@in, FSTOrdTermsWriter.TERMS_CODEC_NAME, FSTOrdTermsWriter.TERMS_VERSION_START, FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void seekDir(org.apache.lucene.store.IndexInput in) throws java.io.IOException
+ private void seekDir(IndexInput @in)
+ {
+ if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
+ {
+ @in.seek(@in.length() - CodecUtil.footerLength() - 8);
+ }
+ else
+ {
+ @in.seek(@in.length() - 8);
+ }
+ @in.seek(@in.readLong());
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void checkFieldSummary(org.apache.lucene.index.SegmentInfo info, org.apache.lucene.store.IndexInput indexIn, org.apache.lucene.store.IndexInput blockIn, org.apache.lucene.codecs.memory.FSTTermsReader.TermsReader field, org.apache.lucene.codecs.memory.FSTTermsReader.TermsReader previous) throws java.io.IOException
+ private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous)
+ {
+ // #docs with field must be <= #docs
+ if (field.docCount < 0 || field.docCount > info.DocCount)
+ {
+ throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.DocCount + " (resource=" + indexIn + ", " + blockIn + ")");
+ }
+ // #postings must be >= #docs with field
+ if (field.sumDocFreq < field.docCount)
+ {
+ throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + indexIn + ", " + blockIn + ")");
+ }
+ // #positions must be >= #postings
+ if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq)
+ {
+ throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + indexIn + ", " + blockIn + ")");
+ }
+ if (previous != null)
+ {
+ throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + indexIn + ", " + blockIn + ")");
+ }
+ }
+
+ public override IEnumerator<string> iterator()
+ {
+ return Collections.unmodifiableSet(fields.Keys).GetEnumerator();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.Terms terms(String field) throws java.io.IOException
+ public override Terms terms(string field)
+ {
+ Debug.Assert(field != null);
+ return fields[field];
+ }
+
+ public override int size()
+ {
+ return fields.Count;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ try
+ {
+ IOUtils.close(postingsReader);
+ }
+ finally
+ {
+ fields.Clear();
+ }
+ }
+
+ internal sealed class TermsReader : Terms
+ {
+ private readonly FSTOrdTermsReader outerInstance;
+
+ internal readonly FieldInfo fieldInfo;
+ internal readonly long numTerms;
+ internal readonly long sumTotalTermFreq;
+ internal readonly long sumDocFreq;
+ internal readonly int docCount;
+ internal readonly int longsSize;
+ internal readonly FST<long?> index;
+
+ internal readonly int numSkipInfo;
+ internal readonly long[] skipInfo;
+ internal readonly sbyte[] statsBlock;
+ internal readonly sbyte[] metaLongsBlock;
+ internal readonly sbyte[] metaBytesBlock;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: org.apache.lucene.codecs.memory.FSTTermsReader.TermsReader(org.apache.lucene.index.FieldInfo fieldInfo, org.apache.lucene.store.IndexInput blockIn, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, org.apache.lucene.util.fst.FST<Long> index) throws java.io.IOException
+ internal TermsReader(FSTOrdTermsReader outerInstance, FieldInfo fieldInfo, IndexInput blockIn, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST<long?> index)
+ {
+ this.outerInstance = outerInstance;
+ this.fieldInfo = fieldInfo;
+ this.numTerms = numTerms;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+ this.longsSize = longsSize;
+ this.index = index;
+
+ assert(numTerms & (~0xffffffffL)) == 0;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int numBlocks = (int)(numTerms + INTERVAL - 1) / INTERVAL;
+ int numBlocks = (int)(numTerms + INTERVAL - 1) / INTERVAL;
+ this.numSkipInfo = longsSize + 3;
+ this.skipInfo = new long[numBlocks * numSkipInfo];
+ this.statsBlock = new sbyte[(int)blockIn.readVLong()];
+ this.metaLongsBlock = new sbyte[(int)blockIn.readVLong()];
+ this.metaBytesBlock = new sbyte[(int)blockIn.readVLong()];
+
+ int last = 0, next = 0;
+ for (int i = 1; i < numBlocks; i++)
+ {
+ next = numSkipInfo * i;
+ for (int j = 0; j < numSkipInfo; j++)
+ {
+ skipInfo[next + j] = skipInfo[last + j] + blockIn.readVLong();
+ }
+ last = next;
+ }
+ blockIn.readBytes(statsBlock, 0, statsBlock.Length);
+ blockIn.readBytes(metaLongsBlock, 0, metaLongsBlock.Length);
+ blockIn.readBytes(metaBytesBlock, 0, metaBytesBlock.Length);
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+ public override bool hasFreqs()
+ {
+ return fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ }
+
+ public override bool hasOffsets()
+ {
+ return fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ public override bool hasPositions()
+ {
+ return fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ public override bool hasPayloads()
+ {
+ return fieldInfo.hasPayloads();
+ }
+
+ public override long size()
+ {
+ return numTerms;
+ }
+
+ public override long SumTotalTermFreq
+ {
+ get
+ {
+ return sumTotalTermFreq;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long getSumDocFreq() throws java.io.IOException
+ public override long SumDocFreq
+ {
+ get
+ {
+ return sumDocFreq;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int getDocCount() throws java.io.IOException
+ public override int DocCount
+ {
+ get
+ {
+ return docCount;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.TermsEnum iterator(org.apache.lucene.index.TermsEnum reuse) throws java.io.IOException
+ public override TermsEnum iterator(TermsEnum reuse)
+ {
+ return new SegmentTermsEnum(this);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.TermsEnum intersect(org.apache.lucene.util.automaton.CompiledAutomaton compiled, org.apache.lucene.util.BytesRef startTerm) throws java.io.IOException
+ public override TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm)
+ {
+ return new IntersectTermsEnum(this, compiled, startTerm);
+ }
+
+ // Only wraps common operations for PBF interact
+ internal abstract class BaseTermsEnum : TermsEnum
+ {
+ private readonly FSTOrdTermsReader.TermsReader outerInstance;
+
+ /* Current term, null when enum ends or unpositioned */
+ internal BytesRef term_Renamed;
+
+ /* Current term's ord, starts from 0 */
+ internal long ord_Renamed;
+
+ /* Current term stats + decoded metadata (customized by PBF) */
+ internal readonly BlockTermState state;
+
+ /* Datainput to load stats & metadata */
+ internal readonly ByteArrayDataInput statsReader = new ByteArrayDataInput();
+ internal readonly ByteArrayDataInput metaLongsReader = new ByteArrayDataInput();
+ internal readonly ByteArrayDataInput metaBytesReader = new ByteArrayDataInput();
+
+ /* To which block is buffered */
+ internal int statsBlockOrd;
+ internal int metaBlockOrd;
+
+ /* Current buffered metadata (long[] & byte[]) */
+ internal long[][] longs;
+ internal int[] bytesStart;
+ internal int[] bytesLength;
+
+ /* Current buffered stats (df & ttf) */
+ internal int[] docFreq_Renamed;
+ internal long[] totalTermFreq_Renamed;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: BaseTermsEnum() throws java.io.IOException
+ internal BaseTermsEnum(FSTOrdTermsReader.TermsReader outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ this.state = outerInstance.outerInstance.postingsReader.newTermState();
+ this.term_Renamed = null;
+ this.statsReader.reset(outerInstance.statsBlock);
+ this.metaLongsReader.reset(outerInstance.metaLongsBlock);
+ this.metaBytesReader.reset(outerInstance.metaBytesBlock);
+
+//JAVA TO C# CONVERTER NOTE: The following call to the 'RectangularArrays' helper class reproduces the rectangular array initialization that is automatic in Java:
+//ORIGINAL LINE: this.longs = new long[INTERVAL][outerInstance.longsSize];
+ this.longs = RectangularArrays.ReturnRectangularLongArray(INTERVAL, outerInstance.longsSize);
+ this.bytesStart = new int[INTERVAL];
+ this.bytesLength = new int[INTERVAL];
+ this.docFreq_Renamed = new int[INTERVAL];
+ this.totalTermFreq_Renamed = new long[INTERVAL];
+ this.statsBlockOrd = -1;
+ this.metaBlockOrd = -1;
+ if (!outerInstance.hasFreqs())
+ {
+ Arrays.fill(totalTermFreq_Renamed, -1);
+ }
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+ /// <summary>
+ /// Decodes stats data into term state </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void decodeStats() throws java.io.IOException
+ internal virtual void decodeStats()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int upto = (int)ord % INTERVAL;
+ int upto = (int)ord_Renamed % INTERVAL;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int oldBlockOrd = statsBlockOrd;
+ int oldBlockOrd = statsBlockOrd;
+ statsBlockOrd = (int)ord_Renamed / INTERVAL;
+ if (oldBlockOrd != statsBlockOrd)
+ {
+ refillStats();
+ }
+ state.docFreq = docFreq_Renamed[upto];
+ state.totalTermFreq = totalTermFreq_Renamed[upto];
+ }
+
+ /// <summary>
+ /// Let PBF decode metadata </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void decodeMetaData() throws java.io.IOException
+ internal virtual void decodeMetaData()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int upto = (int)ord % INTERVAL;
+ int upto = (int)ord_Renamed % INTERVAL;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int oldBlockOrd = metaBlockOrd;
+ int oldBlockOrd = metaBlockOrd;
+ metaBlockOrd = (int)ord_Renamed / INTERVAL;
+ if (metaBlockOrd != oldBlockOrd)
+ {
+ refillMetadata();
+ }
+ metaBytesReader.Position = bytesStart[upto];
+ outerInstance.outerInstance.postingsReader.decodeTerm(longs[upto], metaBytesReader, outerInstance.fieldInfo, state, true);
+ }
+
+ /// <summary>
+ /// Load current stats shard </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: final void refillStats() throws java.io.IOException
+ internal void refillStats()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int offset = statsBlockOrd * numSkipInfo;
+ int offset = statsBlockOrd * outerInstance.numSkipInfo;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int statsFP = (int)skipInfo[offset];
+ int statsFP = (int)outerInstance.skipInfo[offset];
+ statsReader.Position = statsFP;
+ for (int i = 0; i < INTERVAL && !statsReader.eof(); i++)
+ {
+ int code = statsReader.readVInt();
+ if (outerInstance.hasFreqs())
+ {
+ docFreq_Renamed[i] = ((int)((uint)code >> 1));
+ if ((code & 1) == 1)
+ {
+ totalTermFreq_Renamed[i] = docFreq_Renamed[i];
+ }
+ else
+ {
+ totalTermFreq_Renamed[i] = docFreq_Renamed[i] + statsReader.readVLong();
+ }
+ }
+ else
+ {
+ docFreq_Renamed[i] = code;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Load current metadata shard </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: final void refillMetadata() throws java.io.IOException
+ internal void refillMetadata()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int offset = metaBlockOrd * numSkipInfo;
+ int offset = metaBlockOrd * outerInstance.numSkipInfo;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int metaLongsFP = (int)skipInfo[offset + 1];
+ int metaLongsFP = (int)outerInstance.skipInfo[offset + 1];
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int metaBytesFP = (int)skipInfo[offset + 2];
+ int metaBytesFP = (int)outerInstance.skipInfo[offset + 2];
+ metaLongsReader.Position = metaLongsFP;
+ for (int j = 0; j < outerInstance.longsSize; j++)
+ {
+ longs[0][j] = outerInstance.skipInfo[offset + 3 + j] + metaLongsReader.readVLong();
+ }
+ bytesStart[0] = metaBytesFP;
+ bytesLength[0] = (int)metaLongsReader.readVLong();
+ for (int i = 1; i < INTERVAL && !metaLongsReader.eof(); i++)
+ {
+ for (int j = 0; j < outerInstance.longsSize; j++)
+ {
+ longs[i][j] = longs[i - 1][j] + metaLongsReader.readVLong();
+ }
+ bytesStart[i] = bytesStart[i - 1] + bytesLength[i - 1];
+ bytesLength[i] = (int)metaLongsReader.readVLong();
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.TermState termState() throws java.io.IOException
+ public override TermState termState()
+ {
+ decodeMetaData();
+ return state.clone();
+ }
+
+ public override BytesRef term()
+ {
+ return term_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int docFreq() throws java.io.IOException
+ public override int docFreq()
+ {
+ return state.docFreq;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long totalTermFreq() throws java.io.IOException
+ public override long totalTermFreq()
+ {
+ return state.totalTermFreq;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsEnum docs(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsEnum reuse, int flags) throws java.io.IOException
+ public override DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
+ {
+ decodeMetaData();
+ return outerInstance.outerInstance.postingsReader.docs(outerInstance.fieldInfo, state, liveDocs, reuse, flags);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsAndPositionsEnum docsAndPositions(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsAndPositionsEnum reuse, int flags) throws java.io.IOException
+ public override DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags)
+ {
+ if (!outerInstance.hasPositions())
+ {
+ return null;
+ }
+ decodeMetaData();
+ return outerInstance.outerInstance.postingsReader.docsAndPositions(outerInstance.fieldInfo, state, liveDocs, reuse, flags);
+ }
+
+ // TODO: this can be achieved by making use of Util.getByOutput()
+ // and should have related tests
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void seekExact(long ord) throws java.io.IOException
+ public override void seekExact(long ord)
+ {
+ throw new System.NotSupportedException();
+ }
+
+ public override long ord()
+ {
+ throw new System.NotSupportedException();
+ }
+ }
+
+ // Iterates through all terms in this field
+ private sealed class SegmentTermsEnum : BaseTermsEnum
+ {
+ private readonly FSTOrdTermsReader.TermsReader outerInstance;
+
+ internal readonly BytesRefFSTEnum<long?> fstEnum;
+
+ /* True when current term's metadata is decoded */
+ internal bool decoded;
+
+ /* True when current enum is 'positioned' by seekExact(TermState) */
+ internal bool seekPending;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: SegmentTermsEnum() throws java.io.IOException
+ internal SegmentTermsEnum(FSTOrdTermsReader.TermsReader outerInstance) : base(outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ this.fstEnum = new BytesRefFSTEnum<>(outerInstance.index);
+ this.decoded = false;
+ this.seekPending = false;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override void decodeMetaData() throws java.io.IOException
+ internal override void decodeMetaData()
+ {
+ if (!decoded && !seekPending)
+ {
+ base.decodeMetaData();
+ decoded = true;
+ }
+ }
+
+ // Update current enum according to FSTEnum
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void updateEnum(final org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput<Long> pair) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ internal void updateEnum(InputOutput<long?> pair)
+ {
+ if (pair == null)
+ {
+ term_Renamed = null;
+ }
+ else
+ {
+ term_Renamed = pair.input;
+ ord_Renamed = pair.output;
+ decodeStats();
+ }
+ decoded = false;
+ seekPending = false;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+ public override BytesRef next()
+ {
+ if (seekPending) // previously positioned, but termOutputs not fetched
+ {
+ seekPending = false;
+ SeekStatus status = seekCeil(term_Renamed);
+ Debug.Assert(status == SeekStatus.FOUND); // must positioned on valid term
+ }
+ updateEnum(fstEnum.next());
+ return term_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean seekExact(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ public override bool seekExact(BytesRef target)
+ {
+ updateEnum(fstEnum.seekExact(target));
+ return term_Renamed != null;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ public override SeekStatus seekCeil(BytesRef target)
+ {
+ updateEnum(fstEnum.seekCeil(target));
+ if (term_Renamed == null)
+ {
+ return SeekStatus.END;
+ }
+ else
+ {
+ return term_Renamed.Equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
+ }
+ }
+
+ public override void seekExact(BytesRef target, TermState otherState)
+ {
+ if (!target.Equals(term_Renamed))
+ {
+ state.copyFrom(otherState);
+ term_Renamed = BytesRef.deepCopyOf(target);
+ seekPending = true;
+ }
+ }
+ }
+
+ // Iterates intersect result with automaton (cannot seek!)
+ private sealed class IntersectTermsEnum : BaseTermsEnum
+ {
+ private readonly FSTOrdTermsReader.TermsReader outerInstance;
+
+ /* True when current term's metadata is decoded */
+ internal bool decoded;
+
+ /* True when there is pending term when calling next() */
+ internal bool pending;
+
+ /* stack to record how current term is constructed,
+ * used to accumulate metadata or rewind term:
+ * level == term.length + 1,
+ * == 0 when term is null */
+ internal Frame[] stack;
+ internal int level;
+
+ /* term dict fst */
+ internal readonly FST<long?> fst;
+ internal readonly FST.BytesReader fstReader;
+ internal readonly Outputs<long?> fstOutputs;
+
+ /* query automaton to intersect with */
+ internal readonly ByteRunAutomaton fsa;
+
+ private sealed class Frame
+ {
+ private readonly FSTOrdTermsReader.TermsReader.IntersectTermsEnum outerInstance;
+
+ /* fst stats */
+ internal FST.Arc<long?> arc;
+
+ /* automaton stats */
+ internal int state;
+
+ internal Frame(FSTOrdTermsReader.TermsReader.IntersectTermsEnum outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ this.arc = new FST.Arc<>();
+ this.state = -1;
+ }
+
+ public override string ToString()
+ {
+ return "arc=" + arc + " state=" + state;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: IntersectTermsEnum(org.apache.lucene.util.automaton.CompiledAutomaton compiled, org.apache.lucene.util.BytesRef startTerm) throws java.io.IOException
+ internal IntersectTermsEnum(FSTOrdTermsReader.TermsReader outerInstance, CompiledAutomaton compiled, BytesRef startTerm) : base(outerInstance)
+ {
+ //if (TEST) System.out.println("Enum init, startTerm=" + startTerm);
+ this.outerInstance = outerInstance;
+ this.fst = outerInstance.index;
+ this.fstReader = fst.BytesReader;
+ this.fstOutputs = outerInstance.index.outputs;
+ this.fsa = compiled.runAutomaton;
+ this.level = -1;
+ this.stack = new Frame[16];
+ for (int i = 0 ; i < stack.Length; i++)
+ {
+ this.stack[i] = new Frame(this);
+ }
+
+ Frame frame;
+ frame = loadVirtualFrame(newFrame());
+ this.level++;
+ frame = loadFirstFrame(newFrame());
+ pushFrame(frame);
+
+ this.decoded = false;
+ this.pending = false;
+
+ if (startTerm == null)
+ {
+ pending = isAccept(topFrame());
+ }
+ else
+ {
+ doSeekCeil(startTerm);
+ pending = !startTerm.Equals(term_Renamed) && isValid(topFrame()) && isAccept(topFrame());
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override void decodeMetaData() throws java.io.IOException
+ internal override void decodeMetaData()
+ {
+ if (!decoded)
+ {
+ base.decodeMetaData();
+ decoded = true;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override void decodeStats() throws java.io.IOException
+ internal override void decodeStats()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> arc = topFrame().arc;
+ FST.Arc<long?> arc = topFrame().arc;
+ Debug.Assert(arc.nextFinalOutput == fstOutputs.NoOutput);
+ ord_Renamed = arc.output;
+ base.decodeStats();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ public override SeekStatus seekCeil(BytesRef target)
+ {
+ throw new System.NotSupportedException();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+ public override BytesRef next()
+ {
+ //if (TEST) System.out.println("Enum next()");
+ if (pending)
+ {
+ pending = false;
+ decodeStats();
+ return term_Renamed;
+ }
+ decoded = false;
+ while (level > 0)
+ {
+ Frame frame = newFrame();
+ if (loadExpandFrame(topFrame(), frame) != null) // has valid target
+ {
+ pushFrame(frame);
+ if (isAccept(frame)) // gotcha
+ {
+ break;
+ }
+ continue; // check next target
+ }
+ frame = popFrame();
+ while (level > 0)
+ {
+ if (loadNextFrame(topFrame(), frame) != null) // has valid sibling
+ {
+ pushFrame(frame);
+ if (isAccept(frame)) // gotcha
+ {
+ goto DFSBreak;
+ }
+ goto DFSContinue; // check next target
+ }
+ frame = popFrame();
+ }
+ return null;
+ DFSContinue:;
+ }
+ DFSBreak:
+ decodeStats();
+ return term_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: org.apache.lucene.util.BytesRef doSeekCeil(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ internal BytesRef doSeekCeil(BytesRef target)
+ {
+ //if (TEST) System.out.println("Enum doSeekCeil()");
+ Frame frame = null;
+ int label , upto = 0, limit = target.length;
+ while (upto < limit) // to target prefix, or ceil label (rewind prefix)
+ {
+ frame = newFrame();
+ label = target.bytes[upto] & 0xff;
+ frame = loadCeilFrame(label, topFrame(), frame);
+ if (frame == null || frame.arc.label != label)
+ {
+ break;
+ }
+ Debug.Assert(isValid(frame)); // target must be fetched from automaton
+ pushFrame(frame);
+ upto++;
+ }
+ if (upto == limit) // got target
+ {
+ return term_Renamed;
+ }
+ if (frame != null) // got larger term('s prefix)
+ {
+ pushFrame(frame);
+ return isAccept(frame) ? term_Renamed : next();
+ }
+ while (level > 0) // got target's prefix, advance to larger term
+ {
+ frame = popFrame();
+ while (level > 0 && !canRewind(frame))
+ {
+ frame = popFrame();
+ }
+ if (loadNextFrame(topFrame(), frame) != null)
+ {
+ pushFrame(frame);
+ return isAccept(frame) ? term_Renamed : next();
+ }
+ }
+ return null;
+ }
+
+ /// <summary>
+ /// Virtual frame, never pop </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadVirtualFrame(Frame frame) throws java.io.IOException
+ internal Frame loadVirtualFrame(Frame frame)
+ {
+ frame.arc.output = fstOutputs.NoOutput;
+ frame.arc.nextFinalOutput = fstOutputs.NoOutput;
+ frame.state = -1;
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for start arc(node) on fst </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadFirstFrame(Frame frame) throws java.io.IOException
+ internal Frame loadFirstFrame(Frame frame)
+ {
+ frame.arc = fst.getFirstArc(frame.arc);
+ frame.state = fsa.InitialState;
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for target arc(node) on fst </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadExpandFrame(Frame top, Frame frame) throws java.io.IOException
+ internal Frame loadExpandFrame(Frame top, Frame frame)
+ {
+ if (!canGrow(top))
+ {
+ return null;
+ }
+ frame.arc = fst.readFirstRealTargetArc(top.arc.target, frame.arc, fstReader);
+ frame.state = fsa.step(top.state, frame.arc.label);
+ //if (TEST) System.out.println(" loadExpand frame="+frame);
+ if (frame.state == -1)
+ {
+ return loadNextFrame(top, frame);
+ }
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for sibling arc(node) on fst </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadNextFrame(Frame top, Frame frame) throws java.io.IOException
+ internal Frame loadNextFrame(Frame top, Frame frame)
+ {
+ if (!canRewind(frame))
+ {
+ return null;
+ }
+ while (!frame.arc.Last)
+ {
+ frame.arc = fst.readNextRealArc(frame.arc, fstReader);
+ frame.state = fsa.step(top.state, frame.arc.label);
+ if (frame.state != -1)
+ {
+ break;
+ }
+ }
+ //if (TEST) System.out.println(" loadNext frame="+frame);
+ if (frame.state == -1)
+ {
+ return null;
+ }
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for target arc(node) on fst, so that
+ /// arc.label >= label and !fsa.reject(arc.label)
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadCeilFrame(int label, Frame top, Frame frame) throws java.io.IOException
+ internal Frame loadCeilFrame(int label, Frame top, Frame frame)
+ {
+ FST.Arc<long?> arc = frame.arc;
+ arc = Util.readCeilArc(label, fst, top.arc, arc, fstReader);
+ if (arc == null)
+ {
+ return null;
+ }
+ frame.state = fsa.step(top.state, arc.label);
+ //if (TEST) System.out.println(" loadCeil frame="+frame);
+ if (frame.state == -1)
+ {
+ return loadNextFrame(top, frame);
+ }
+ return frame;
+ }
+
+ internal bool isAccept(Frame frame) // reach a term both fst&fsa accepts
+ {
+ return fsa.isAccept(frame.state) && frame.arc.Final;
+ }
+ internal bool isValid(Frame frame) // reach a prefix both fst&fsa won't reject
+ {
+ return frame.state != -1; //frame != null &&
+ }
+ internal bool canGrow(Frame frame) // can walk forward on both fst&fsa
+ {
+ return frame.state != -1 && FST.targetHasArcs(frame.arc);
+ }
+ internal bool canRewind(Frame frame) // can jump to sibling
+ {
+ return !frame.arc.Last;
+ }
+
+ internal void pushFrame(Frame frame)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<Long> arc = frame.arc;
+ FST.Arc<long?> arc = frame.arc;
+ arc.output = fstOutputs.add(topFrame().arc.output, arc.output);
+ term_Renamed = grow(arc.label);
+ level++;
+ Debug.Assert(frame == stack[level]);
+ }
+
+ internal Frame popFrame()
+ {
+ term_Renamed = shrink();
+ return stack[level--];
+ }
+
+ internal Frame newFrame()
+ {
+ if (level + 1 == stack.Length)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Frame[] temp = new Frame[org.apache.lucene.util.ArrayUtil.oversize(level+2, org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ Frame[] temp = new Frame[ArrayUtil.oversize(level + 2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ Array.Copy(stack, 0, temp, 0, stack.Length);
+ for (int i = stack.Length; i < temp.Length; i++)
+ {
+ temp[i] = new Frame(this);
+ }
+ stack = temp;
+ }
+ return stack[level + 1];
+ }
+
+ internal Frame topFrame()
+ {
+ return stack[level];
+ }
+
+ internal BytesRef grow(int label)
+ {
+ if (term_Renamed == null)
+ {
+ term_Renamed = new BytesRef(new sbyte[16], 0, 0);
+ }
+ else
+ {
+ if (term_Renamed.length == term_Renamed.bytes.length)
+ {
+ term_Renamed.grow(term_Renamed.length + 1);
+ }
+ term_Renamed.bytes[term_Renamed.length++] = (sbyte)label;
+ }
+ return term_Renamed;
+ }
+
+ internal BytesRef shrink()
+ {
+ if (term_Renamed.length == 0)
+ {
+ term_Renamed = null;
+ }
+ else
+ {
+ term_Renamed.length--;
+ }
+ return term_Renamed;
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: static<T> void walk(org.apache.lucene.util.fst.FST<T> fst) throws java.io.IOException
+ internal static void walk<T>(FST<T> fst)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.ArrayList<org.apache.lucene.util.fst.FST.Arc<T>> queue = new java.util.ArrayList<>();
+ List<FST.Arc<T>> queue = new List<FST.Arc<T>>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.BitSet seen = new java.util.BitSet();
+ BitArray seen = new BitArray();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.BytesReader reader = fst.getBytesReader();
+ FST.BytesReader reader = fst.BytesReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<T> startArc = fst.getFirstArc(new org.apache.lucene.util.fst.FST.Arc<T>());
+ FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());
+ queue.Add(startArc);
+ while (queue.Count > 0)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<T> arc = queue.remove(0);
+ FST.Arc<T> arc = queue.Remove(0);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long node = arc.target;
+ long node = arc.target;
+ //System.out.println(arc);
+ if (FST.targetHasArcs(arc) && !seen.Get((int) node))
+ {
+ seen.Set((int) node, true);
+ fst.readFirstRealTargetArc(node, arc, reader);
+ while (true)
+ {
+ queue.Add((new FST.Arc<T>()).copyFrom(arc));
+ if (arc.Last)
+ {
+ break;
+ }
+ else
+ {
+ fst.readNextRealArc(arc, reader);
+ }
+ }
+ }
+ }
+ }
+
+ public override long ramBytesUsed()
+ {
+ long ramBytesUsed = 0;
+ foreach (TermsReader r in fields.Values)
+ {
+ if (r.index != null)
+ {
+ ramBytesUsed += r.index.sizeInBytes();
+ ramBytesUsed += RamUsageEstimator.sizeOf(r.metaBytesBlock);
+ ramBytesUsed += RamUsageEstimator.sizeOf(r.metaLongsBlock);
+ ramBytesUsed += RamUsageEstimator.sizeOf(r.skipInfo);
+ ramBytesUsed += RamUsageEstimator.sizeOf(r.statsBlock);
+ }
+ }
+ return ramBytesUsed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+ public override void checkIntegrity()
+ {
+ postingsReader.checkIntegrity();
+ }
+ }
+
+}
\ No newline at end of file
[10/10] git commit: Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
Lucene.Net.Codes/Sep fully ported, work done on SimpleText and Memory as well
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/d852d5b0
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/d852d5b0
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/d852d5b0
Branch: refs/heads/master
Commit: d852d5b0479b24e84f66a6d24a0136fc7d5562ae
Parents: 3b226e8
Author: Prescott Nasser <pn...@apache.org>
Authored: Sun Sep 28 01:49:21 2014 -0700
Committer: Prescott Nasser <pn...@apache.org>
Committed: Sun Sep 28 01:49:21 2014 -0700
----------------------------------------------------------------------
src/Lucene.Net.Codecs/HashMapHelperClass.cs | 26 +
src/Lucene.Net.Codecs/Lucene.Net.Codecs.csproj | 3 +
.../Memory/DirectDocValuesConsumer.cs | 727 ++-
.../Memory/DirectDocValuesFormat.cs | 161 +-
.../Memory/DirectDocValuesProducer.cs | 1271 +++--
.../Memory/DirectPostingsFormat.cs | 5238 ++++++++++--------
.../Memory/FSTOrdPostingsFormat.cs | 151 +-
.../Memory/FSTOrdPulsing41PostingsFormat.cs | 174 +-
.../Memory/FSTOrdTermsReader.cs | 2014 ++++---
.../Memory/FSTOrdTermsWriter.cs | 809 +--
.../Memory/FSTPostingsFormat.cs | 151 +-
.../Memory/FSTPulsing41PostingsFormat.cs | 176 +-
src/Lucene.Net.Codecs/Memory/FSTTermOutputs.cs | 704 ++-
src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs | 1772 +++---
src/Lucene.Net.Codecs/Memory/FSTTermsWriter.cs | 568 +-
.../Memory/MemoryDocValuesConsumer.cs | 956 ++--
.../Memory/MemoryDocValuesFormat.cs | 133 +-
.../Memory/MemoryDocValuesProducer.cs | 1653 +++---
.../Memory/MemoryPostingsFormat.cs | 2139 +++----
src/Lucene.Net.Codecs/RectangularArrays.cs | 29 +
src/Lucene.Net.Codecs/Sep/IntIndexInput.cs | 82 +-
src/Lucene.Net.Codecs/Sep/IntIndexOutput.cs | 80 +-
src/Lucene.Net.Codecs/Sep/IntStreamFactory.cs | 40 +-
src/Lucene.Net.Codecs/Sep/SepPostingsReader.cs | 1402 ++---
src/Lucene.Net.Codecs/Sep/SepPostingsWriter.cs | 728 +--
src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs | 389 +-
src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs | 391 +-
.../SimpleText/SimpleTextDocValuesReader.cs | 27 +-
.../SimpleText/SimpleTextDocValuesWriter.cs | 74 +-
.../SimpleText/SimpleTextSegmentInfoReader.cs | 4 +-
.../SimpleText/SimpleTextTermVectorsWriter.cs | 50 +-
src/Lucene.Net.Codecs/StringHelperClass.cs | 6 +-
.../Codecs/StoredFieldsReader.cs | 46 +-
33 files changed, 12444 insertions(+), 9730 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/HashMapHelperClass.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/HashMapHelperClass.cs b/src/Lucene.Net.Codecs/HashMapHelperClass.cs
new file mode 100644
index 0000000..489b446
--- /dev/null
+++ b/src/Lucene.Net.Codecs/HashMapHelperClass.cs
@@ -0,0 +1,26 @@
+//---------------------------------------------------------------------------------------------------------
+// Copyright © 2007 - 2014 Tangible Software Solutions Inc.
+// This class can be used by anyone provided that the copyright notice remains intact.
+//
+// This class is used to replace calls to some Java HashMap or Hashtable methods.
+//---------------------------------------------------------------------------------------------------------
+using System.Collections.Generic;
+internal static class HashMapHelperClass
+{
+ internal static HashSet<KeyValuePair<TKey, TValue>> SetOfKeyValuePairs<TKey, TValue>(this IDictionary<TKey, TValue> dictionary)
+ {
+ HashSet<KeyValuePair<TKey, TValue>> entries = new HashSet<KeyValuePair<TKey, TValue>>();
+ foreach (KeyValuePair<TKey, TValue> keyValuePair in dictionary)
+ {
+ entries.Add(keyValuePair);
+ }
+ return entries;
+ }
+
+ internal static TValue GetValueOrNull<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key)
+ {
+ TValue ret;
+ dictionary.TryGetValue(key, out ret);
+ return ret;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Lucene.Net.Codecs.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Lucene.Net.Codecs.csproj b/src/Lucene.Net.Codecs/Lucene.Net.Codecs.csproj
index 31340ae..f9500ef 100644
--- a/src/Lucene.Net.Codecs/Lucene.Net.Codecs.csproj
+++ b/src/Lucene.Net.Codecs/Lucene.Net.Codecs.csproj
@@ -33,6 +33,7 @@
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
+ <Reference Include="System.Numerics" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
@@ -60,6 +61,7 @@
<Compile Include="DiskDV\DiskDocValuesFormat.cs" />
<Compile Include="DiskDV\DiskDocValuesProducer.cs" />
<Compile Include="DiskDV\DiskNormsFormat.cs" />
+ <Compile Include="HashMapHelperClass.cs" />
<Compile Include="Intblock\FixedIntBlockIndexInput.cs" />
<Compile Include="Intblock\FixedIntBlockIndexOutput.cs" />
<Compile Include="Intblock\IBlockReader.cs" />
@@ -89,6 +91,7 @@
<Compile Include="Pulsing\PulsingPostingsFormat.cs" />
<Compile Include="Pulsing\PulsingPostingsReader.cs" />
<Compile Include="Pulsing\PulsingPostingsWriter.cs" />
+ <Compile Include="RectangularArrays.cs" />
<Compile Include="Sep\IntIndexInput.cs" />
<Compile Include="Sep\IntIndexOutput.cs" />
<Compile Include="Sep\IntStreamFactory.cs" />
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs b/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
index ddddf6f..8362727 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
@@ -1,304 +1,423 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import codecs.CodecUtil;
-import codecs.DocValuesConsumer;
-import index.FieldInfo;
-import index.IndexFileNames;
-import index.SegmentWriteState;
-import store.IndexOutput;
-import util.BytesRef;
-import util.IOUtils;
-
-import static codecs.memory.DirectDocValuesProducer.VERSION_CURRENT;
-import static codecs.memory.DirectDocValuesProducer.BYTES;
-import static codecs.memory.DirectDocValuesProducer.SORTED;
-import static codecs.memory.DirectDocValuesProducer.SORTED_SET;
-import static codecs.memory.DirectDocValuesProducer.NUMBER;
-
-/**
- * Writer for {@link DirectDocValuesFormat}
- */
-
-class DirectDocValuesConsumer extends DocValuesConsumer {
- IndexOutput data, meta;
- final int maxDoc;
-
- DirectDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) {
- maxDoc = state.segmentInfo.getDocCount();
- bool success = false;
- try {
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- @Override
- public void addNumericField(FieldInfo field, Iterable<Number> values) {
- meta.writeVInt(field.number);
- meta.writeByte(NUMBER);
- addNumericFieldValues(field, values);
- }
-
- private void addNumericFieldValues(FieldInfo field, Iterable<Number> values) {
- meta.writeLong(data.getFilePointer());
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
- bool missing = false;
-
- long count = 0;
- for (Number nv : values) {
- if (nv != null) {
- long v = nv.longValue();
- minValue = Math.min(minValue, v);
- maxValue = Math.max(maxValue, v);
- } else {
- missing = true;
- }
- count++;
- if (count >= DirectDocValuesFormat.MAX_SORTED_SET_ORDS) {
- throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + DirectDocValuesFormat.MAX_SORTED_SET_ORDS + " values/total ords");
- }
- }
- meta.writeInt((int) count);
-
- if (missing) {
- long start = data.getFilePointer();
- writeMissingBitset(values);
- meta.writeLong(start);
- meta.writeLong(data.getFilePointer() - start);
- } else {
- meta.writeLong(-1L);
- }
-
- byte byteWidth;
- if (minValue >= Byte.MIN_VALUE && maxValue <= Byte.MAX_VALUE) {
- byteWidth = 1;
- } else if (minValue >= Short.MIN_VALUE && maxValue <= Short.MAX_VALUE) {
- byteWidth = 2;
- } else if (minValue >= Integer.MIN_VALUE && maxValue <= Integer.MAX_VALUE) {
- byteWidth = 4;
- } else {
- byteWidth = 8;
- }
- meta.writeByte(byteWidth);
-
- for (Number nv : values) {
- long v;
- if (nv != null) {
- v = nv.longValue();
- } else {
- v = 0;
- }
-
- switch(byteWidth) {
- case 1:
- data.writeByte((byte) v);
- break;
- case 2:
- data.writeShort((short) v);
- break;
- case 4:
- data.writeInt((int) v);
- break;
- case 8:
- data.writeLong(v);
- break;
- }
- }
- }
-
- @Override
- public void close() {
- bool success = false;
- try {
- if (meta != null) {
- meta.writeVInt(-1); // write EOF marker
- CodecUtil.writeFooter(meta); // write checksum
- }
- if (data != null) {
- CodecUtil.writeFooter(data);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(data, meta);
- } else {
- IOUtils.closeWhileHandlingException(data, meta);
- }
- data = meta = null;
- }
- }
-
- @Override
- public void addBinaryField(FieldInfo field, final Iterable<BytesRef> values) {
- meta.writeVInt(field.number);
- meta.writeByte(BYTES);
- addBinaryFieldValues(field, values);
- }
-
- private void addBinaryFieldValues(FieldInfo field, final Iterable<BytesRef> values) {
- // write the byte[] data
- final long startFP = data.getFilePointer();
- bool missing = false;
- long totalBytes = 0;
- int count = 0;
- for(BytesRef v : values) {
- if (v != null) {
- data.writeBytes(v.bytes, v.offset, v.length);
- totalBytes += v.length;
- if (totalBytes > DirectDocValuesFormat.MAX_TOTAL_BYTES_LENGTH) {
- throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, cannot have more than DirectDocValuesFormat.MAX_TOTAL_BYTES_LENGTH (" + DirectDocValuesFormat.MAX_TOTAL_BYTES_LENGTH + ") bytes");
- }
- } else {
- missing = true;
- }
- count++;
- }
-
- meta.writeLong(startFP);
- meta.writeInt((int) totalBytes);
- meta.writeInt(count);
- if (missing) {
- long start = data.getFilePointer();
- writeMissingBitset(values);
- meta.writeLong(start);
- meta.writeLong(data.getFilePointer() - start);
- } else {
- meta.writeLong(-1L);
- }
-
- int addr = 0;
- for (BytesRef v : values) {
- data.writeInt(addr);
- if (v != null) {
- addr += v.length;
- }
- }
- data.writeInt(addr);
- }
-
- // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on,
- // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode)
- void writeMissingBitset(Iterable<?> values) {
- long bits = 0;
- int count = 0;
- for (Object v : values) {
- if (count == 64) {
- data.writeLong(bits);
- count = 0;
- bits = 0;
- }
- if (v != null) {
- bits |= 1L << (count & 0x3f);
- }
- count++;
- }
- if (count > 0) {
- data.writeLong(bits);
- }
- }
-
- @Override
- public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) {
- meta.writeVInt(field.number);
- meta.writeByte(SORTED);
-
- // write the ordinals as numerics
- addNumericFieldValues(field, docToOrd);
-
- // write the values as binary
- addBinaryFieldValues(field, values);
- }
-
- // note: this might not be the most efficient... but its fairly simple
- @Override
- public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) {
- meta.writeVInt(field.number);
- meta.writeByte(SORTED_SET);
-
- // First write docToOrdCounts, except we "aggregate" the
- // counts so they turn into addresses, and add a final
- // value = the total aggregate:
- addNumericFieldValues(field, new Iterable<Number>() {
-
- // Just aggregates the count values so they become
- // "addresses", and adds one more value in the end
- // (the final sum):
-
- @Override
- public Iterator<Number> iterator() {
- final Iterator<Number> iter = docToOrdCount.iterator();
-
- return new Iterator<Number>() {
-
- long sum;
- bool ended;
-
- @Override
- public bool hasNext() {
- return iter.hasNext() || !ended;
- }
-
- @Override
- public Number next() {
- long toReturn = sum;
-
- if (iter.hasNext()) {
- Number n = iter.next();
- if (n != null) {
- sum += n.longValue();
- }
- } else if (!ended) {
- ended = true;
- } else {
- Debug.Assert( false;
- }
-
- return toReturn;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- });
-
- // Write ordinals for all docs, appended into one big
- // numerics:
- addNumericFieldValues(field, ords);
-
- // write the values as binary
- addBinaryFieldValues(field, values);
- }
-}
+using System;
+using System.Diagnostics;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using IndexOutput = org.apache.lucene.store.IndexOutput;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.VERSION_CURRENT;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.BYTES;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.SORTED_SET;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to .NET:
+ import static org.apache.lucene.codecs.memory.DirectDocValuesProducer.NUMBER;
+
+ /// <summary>
+ /// Writer for <seealso cref="DirectDocValuesFormat"/>
+ /// </summary>
+
+ internal class DirectDocValuesConsumer : DocValuesConsumer
+ {
+ internal IndexOutput data, meta;
+ internal readonly int maxDoc;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: DirectDocValuesConsumer(org.apache.lucene.index.SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws java.io.IOException
+ internal DirectDocValuesConsumer(SegmentWriteState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
+ {
+ maxDoc = state.segmentInfo.DocCount;
+ bool success = false;
+ try
+ {
+ string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+ data = state.directory.createOutput(dataName, state.context);
+ CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+ meta = state.directory.createOutput(metaName, state.context);
+ CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(this);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addNumericField(org.apache.lucene.index.FieldInfo field, Iterable<Number> values) throws java.io.IOException
+ public override void addNumericField(FieldInfo field, IEnumerable<Number> values)
+ {
+ meta.writeVInt(field.number);
+ meta.writeByte(NUMBER);
+ addNumericFieldValues(field, values);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void addNumericFieldValues(org.apache.lucene.index.FieldInfo field, Iterable<Number> values) throws java.io.IOException
+ private void addNumericFieldValues(FieldInfo field, IEnumerable<Number> values)
+ {
+ meta.writeLong(data.FilePointer);
+ long minValue = long.MaxValue;
+ long maxValue = long.MinValue;
+ bool missing = false;
+
+ long count = 0;
+ foreach (Number nv in values)
+ {
+ if (nv != null)
+ {
+ long v = (long)nv;
+ minValue = Math.Min(minValue, v);
+ maxValue = Math.Max(maxValue, v);
+ }
+ else
+ {
+ missing = true;
+ }
+ count++;
+ if (count >= DirectDocValuesFormat.MAX_SORTED_SET_ORDS)
+ {
+ throw new System.ArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + DirectDocValuesFormat.MAX_SORTED_SET_ORDS + " values/total ords");
+ }
+ }
+ meta.writeInt((int) count);
+
+ if (missing)
+ {
+ long start = data.FilePointer;
+ writeMissingBitset(values);
+ meta.writeLong(start);
+ meta.writeLong(data.FilePointer - start);
+ }
+ else
+ {
+ meta.writeLong(-1L);
+ }
+
+ sbyte byteWidth;
+ if (minValue >= sbyte.MinValue && maxValue <= sbyte.MaxValue)
+ {
+ byteWidth = 1;
+ }
+ else if (minValue >= short.MinValue && maxValue <= short.MaxValue)
+ {
+ byteWidth = 2;
+ }
+ else if (minValue >= int.MinValue && maxValue <= int.MaxValue)
+ {
+ byteWidth = 4;
+ }
+ else
+ {
+ byteWidth = 8;
+ }
+ meta.writeByte(byteWidth);
+
+ foreach (Number nv in values)
+ {
+ long v;
+ if (nv != null)
+ {
+ v = (long)nv;
+ }
+ else
+ {
+ v = 0;
+ }
+
+ switch (byteWidth)
+ {
+ case 1:
+ data.writeByte((sbyte) v);
+ break;
+ case 2:
+ data.writeShort((short) v);
+ break;
+ case 4:
+ data.writeInt((int) v);
+ break;
+ case 8:
+ data.writeLong(v);
+ break;
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ bool success = false;
+ try
+ {
+ if (meta != null)
+ {
+ meta.writeVInt(-1); // write EOF marker
+ CodecUtil.writeFooter(meta); // write checksum
+ }
+ if (data != null)
+ {
+ CodecUtil.writeFooter(data);
+ }
+ success = true;
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.close(data, meta);
+ }
+ else
+ {
+ IOUtils.closeWhileHandlingException(data, meta);
+ }
+ data = meta = null;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addBinaryField(org.apache.lucene.index.FieldInfo field, final Iterable<org.apache.lucene.util.BytesRef> values) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ public override void addBinaryField(FieldInfo field, IEnumerable<BytesRef> values)
+ {
+ meta.writeVInt(field.number);
+ meta.writeByte(BYTES);
+ addBinaryFieldValues(field, values);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void addBinaryFieldValues(org.apache.lucene.index.FieldInfo field, final Iterable<org.apache.lucene.util.BytesRef> values) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ private void addBinaryFieldValues(FieldInfo field, IEnumerable<BytesRef> values)
+ {
+ // write the byte[] data
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long startFP = data.getFilePointer();
+ long startFP = data.FilePointer;
+ bool missing = false;
+ long totalBytes = 0;
+ int count = 0;
+ foreach (BytesRef v in values)
+ {
+ if (v != null)
+ {
+ data.writeBytes(v.bytes, v.offset, v.length);
+ totalBytes += v.length;
+ if (totalBytes > DirectDocValuesFormat.MAX_TOTAL_BYTES_LENGTH)
+ {
+ throw new System.ArgumentException("DocValuesField \"" + field.name + "\" is too large, cannot have more than DirectDocValuesFormat.MAX_TOTAL_BYTES_LENGTH (" + DirectDocValuesFormat.MAX_TOTAL_BYTES_LENGTH + ") bytes");
+ }
+ }
+ else
+ {
+ missing = true;
+ }
+ count++;
+ }
+
+ meta.writeLong(startFP);
+ meta.writeInt((int) totalBytes);
+ meta.writeInt(count);
+ if (missing)
+ {
+ long start = data.FilePointer;
+ writeMissingBitset(values);
+ meta.writeLong(start);
+ meta.writeLong(data.FilePointer - start);
+ }
+ else
+ {
+ meta.writeLong(-1L);
+ }
+
+ int addr = 0;
+ foreach (BytesRef v in values)
+ {
+ data.writeInt(addr);
+ if (v != null)
+ {
+ addr += v.length;
+ }
+ }
+ data.writeInt(addr);
+ }
+
+ // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on,
+ // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode)
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void writeMissingBitset(Iterable<?> values) throws java.io.IOException
+ internal virtual void writeMissingBitset<T1>(IEnumerable<T1> values)
+ {
+ long bits = 0;
+ int count = 0;
+ foreach (object v in values)
+ {
+ if (count == 64)
+ {
+ data.writeLong(bits);
+ count = 0;
+ bits = 0;
+ }
+ if (v != null)
+ {
+ bits |= 1L << (count & 0x3f);
+ }
+ count++;
+ }
+ if (count > 0)
+ {
+ data.writeLong(bits);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addSortedField(org.apache.lucene.index.FieldInfo field, Iterable<org.apache.lucene.util.BytesRef> values, Iterable<Number> docToOrd) throws java.io.IOException
+ public override void addSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<Number> docToOrd)
+ {
+ meta.writeVInt(field.number);
+ meta.writeByte(SORTED);
+
+ // write the ordinals as numerics
+ addNumericFieldValues(field, docToOrd);
+
+ // write the values as binary
+ addBinaryFieldValues(field, values);
+ }
+
+ // note: this might not be the most efficient... but its fairly simple
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void addSortedSetField(org.apache.lucene.index.FieldInfo field, Iterable<org.apache.lucene.util.BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ public override void addSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<Number> docToOrdCount, IEnumerable<Number> ords)
+ {
+ meta.writeVInt(field.number);
+ meta.writeByte(SORTED_SET);
+
+ // First write docToOrdCounts, except we "aggregate" the
+ // counts so they turn into addresses, and add a final
+ // value = the total aggregate:
+ addNumericFieldValues(field, new IterableAnonymousInnerClassHelper(this, docToOrdCount));
+
+ // Write ordinals for all docs, appended into one big
+ // numerics:
+ addNumericFieldValues(field, ords);
+
+ // write the values as binary
+ addBinaryFieldValues(field, values);
+ }
+
+ private class IterableAnonymousInnerClassHelper : IEnumerable<Number>
+ {
+ private readonly DirectDocValuesConsumer outerInstance;
+
+ private IEnumerable<Number> docToOrdCount;
+
+ public IterableAnonymousInnerClassHelper(DirectDocValuesConsumer outerInstance, IEnumerable<Number> docToOrdCount)
+ {
+ this.outerInstance = outerInstance;
+ this.docToOrdCount = docToOrdCount;
+ }
+
+
+ // Just aggregates the count values so they become
+ // "addresses", and adds one more value in the end
+ // (the final sum):
+
+ public virtual IEnumerator<Number> GetEnumerator()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.Iterator<Number> iter = docToOrdCount.iterator();
+ IEnumerator<Number> iter = docToOrdCount.GetEnumerator();
+
+ return new IteratorAnonymousInnerClassHelper(this, iter);
+ }
+
+ private class IteratorAnonymousInnerClassHelper : IEnumerator<Number>
+ {
+ private readonly IterableAnonymousInnerClassHelper outerInstance;
+
+ private IEnumerator<Number> iter;
+
+ public IteratorAnonymousInnerClassHelper(IterableAnonymousInnerClassHelper outerInstance, IEnumerator<Number> iter)
+ {
+ this.outerInstance = outerInstance;
+ this.iter = iter;
+ }
+
+
+ internal long sum;
+ internal bool ended;
+
+ public virtual bool hasNext()
+ {
+//JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops:
+ return iter.hasNext() || !ended;
+ }
+
+ public virtual Number next()
+ {
+ long toReturn = sum;
+
+//JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops:
+ if (iter.hasNext())
+ {
+//JAVA TO C# CONVERTER TODO TASK: Java iterators are only converted within the context of 'while' and 'for' loops:
+ Number n = iter.next();
+ if (n != null)
+ {
+ sum += (long)n;
+ }
+ }
+ else if (!ended)
+ {
+ ended = true;
+ }
+ else
+ {
+ Debug.Assert(false);
+ }
+
+ return toReturn;
+ }
+
+ public virtual void remove()
+ {
+ throw new System.NotSupportedException();
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/DirectDocValuesFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectDocValuesFormat.cs b/src/Lucene.Net.Codecs/Memory/DirectDocValuesFormat.cs
index fb98c99..715d190 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectDocValuesFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectDocValuesFormat.cs
@@ -1,83 +1,94 @@
-package codecs.memory;
+namespace org.apache.lucene.codecs.memory
+{
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.io.IOException;
+ using SortedSetDocValuesField = org.apache.lucene.document.SortedSetDocValuesField; // javadocs
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
-import codecs.DocValuesConsumer;
-import codecs.DocValuesFormat;
-import codecs.DocValuesProducer;
-import document.SortedSetDocValuesField; // javadocs
-import index.SegmentReadState;
-import index.SegmentWriteState;
-import util.ArrayUtil;
+ /// <summary>
+ /// In-memory docvalues format that does no (or very little)
+ /// compression. Indexed values are stored on disk, but
+ /// then at search time all values are loaded into memory as
+ /// simple java arrays. For numeric values, it uses
+ /// byte[], short[], int[], long[] as necessary to fit the
+ /// range of the values. For binary values, there is an int
+ /// (4 bytes) overhead per value.
+ ///
+ /// <para>Limitations:
+ /// <ul>
+ /// <li>For binary and sorted fields the total space
+ /// required for all binary values cannot exceed about
+ /// 2.1 GB (see #MAX_TOTAL_BYTES_LENGTH).</li>
+ ///
+ /// <li>For sorted set fields, the sum of the size of each
+ /// document's set of values cannot exceed about 2.1 B
+ /// values (see #MAX_SORTED_SET_ORDS). For example,
+ /// if every document has 10 values (10 instances of
+ /// <seealso cref="SortedSetDocValuesField"/>) added, then no
+ /// more than ~210 M documents can be added to one
+ /// segment. </li>
+ /// </ul>
+ /// </para>
+ /// </summary>
-/** In-memory docvalues format that does no (or very little)
- * compression. Indexed values are stored on disk, but
- * then at search time all values are loaded into memory as
- * simple java arrays. For numeric values, it uses
- * byte[], short[], int[], long[] as necessary to fit the
- * range of the values. For binary values, there is an int
- * (4 bytes) overhead per value.
- *
- * <p>Limitations:
- * <ul>
- * <li>For binary and sorted fields the total space
- * required for all binary values cannot exceed about
- * 2.1 GB (see #MAX_TOTAL_BYTES_LENGTH).</li>
- *
- * <li>For sorted set fields, the sum of the size of each
- * document's set of values cannot exceed about 2.1 B
- * values (see #MAX_SORTED_SET_ORDS). For example,
- * if every document has 10 values (10 instances of
- * {@link SortedSetDocValuesField}) added, then no
- * more than ~210 M documents can be added to one
- * segment. </li>
- * </ul> */
+ public class DirectDocValuesFormat : DocValuesFormat
+ {
-public class DirectDocValuesFormat extends DocValuesFormat {
+ /// <summary>
+ /// The sum of all byte lengths for binary field, or for
+ /// the unique values in sorted or sorted set fields, cannot
+ /// exceed this.
+ /// </summary>
+ public static readonly int MAX_TOTAL_BYTES_LENGTH = ArrayUtil.MAX_ARRAY_LENGTH;
- /** The sum of all byte lengths for binary field, or for
- * the unique values in sorted or sorted set fields, cannot
- * exceed this. */
- public final static int MAX_TOTAL_BYTES_LENGTH = ArrayUtil.MAX_ARRAY_LENGTH;
+ /// <summary>
+ /// The sum of the number of values across all documents
+ /// in a sorted set field cannot exceed this.
+ /// </summary>
+ public static readonly int MAX_SORTED_SET_ORDS = ArrayUtil.MAX_ARRAY_LENGTH;
- /** The sum of the number of values across all documents
- * in a sorted set field cannot exceed this. */
- public final static int MAX_SORTED_SET_ORDS = ArrayUtil.MAX_ARRAY_LENGTH;
+ /// <summary>
+ /// Sole constructor. </summary>
+ public DirectDocValuesFormat() : base("Direct")
+ {
+ }
- /** Sole constructor. */
- public DirectDocValuesFormat() {
- super("Direct");
- }
-
- @Override
- public DocValuesConsumer fieldsConsumer(SegmentWriteState state) {
- return new DirectDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
- }
-
- @Override
- public DocValuesProducer fieldsProducer(SegmentReadState state) {
- return new DirectDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
- }
-
- static final String DATA_CODEC = "DirectDocValuesData";
- static final String DATA_EXTENSION = "dvdd";
- static final String METADATA_CODEC = "DirectDocValuesMetadata";
- static final String METADATA_EXTENSION = "dvdm";
-}
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.DocValuesConsumer fieldsConsumer(org.apache.lucene.index.SegmentWriteState state) throws java.io.IOException
+ public override DocValuesConsumer fieldsConsumer(SegmentWriteState state)
+ {
+ return new DirectDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.DocValuesProducer fieldsProducer(org.apache.lucene.index.SegmentReadState state) throws java.io.IOException
+ public override DocValuesProducer fieldsProducer(SegmentReadState state)
+ {
+ return new DirectDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
+
+ internal const string DATA_CODEC = "DirectDocValuesData";
+ internal const string DATA_EXTENSION = "dvdd";
+ internal const string METADATA_CODEC = "DirectDocValuesMetadata";
+ internal const string METADATA_EXTENSION = "dvdm";
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs b/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
index 6d63d5b..ee88b68 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
@@ -1,511 +1,760 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import codecs.CodecUtil;
-import codecs.DocValuesProducer;
-import index.BinaryDocValues;
-import index.CorruptIndexException;
-import index.DocValues;
-import index.FieldInfo;
-import index.IndexFileNames;
-import index.NumericDocValues;
-import index.RandomAccessOrds;
-import index.SegmentReadState;
-import index.SortedDocValues;
-import index.SortedSetDocValues;
-import store.ChecksumIndexInput;
-import store.IndexInput;
-import util.Bits;
-import util.BytesRef;
-import util.FixedBitSet;
-import util.IOUtils;
-import util.RamUsageEstimator;
-
-/**
- * Reader for {@link DirectDocValuesFormat}
- */
-
-class DirectDocValuesProducer extends DocValuesProducer {
- // metadata maps (just file pointers and minimal stuff)
- private final Map<Integer,NumericEntry> numerics = new HashMap<>();
- private final Map<Integer,BinaryEntry> binaries = new HashMap<>();
- private final Map<Integer,SortedEntry> sorteds = new HashMap<>();
- private final Map<Integer,SortedSetEntry> sortedSets = new HashMap<>();
- private final IndexInput data;
-
- // ram instances we have already loaded
- private final Map<Integer,NumericDocValues> numericInstances =
- new HashMap<>();
- private final Map<Integer,BinaryDocValues> binaryInstances =
- new HashMap<>();
- private final Map<Integer,SortedDocValues> sortedInstances =
- new HashMap<>();
- private final Map<Integer,SortedSetRawValues> sortedSetInstances =
- new HashMap<>();
- private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<>();
-
- private final int maxDoc;
- private final AtomicLong ramBytesUsed;
- private final int version;
-
- static final byte NUMBER = 0;
- static final byte BYTES = 1;
- static final byte SORTED = 2;
- static final byte SORTED_SET = 3;
-
- static final int VERSION_START = 0;
- static final int VERSION_CHECKSUM = 1;
- static final int VERSION_CURRENT = VERSION_CHECKSUM;
-
- DirectDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) {
- maxDoc = state.segmentInfo.getDocCount();
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- // read in the entries from the metadata file.
- ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
- ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
- bool success = false;
- try {
- version = CodecUtil.checkHeader(in, metaCodec,
- VERSION_START,
- VERSION_CURRENT);
- readFields(in);
-
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checkFooter(in);
- } else {
- CodecUtil.checkEOF(in);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
- }
- }
-
- success = false;
- try {
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = state.directory.openInput(dataName, state.context);
- final int version2 = CodecUtil.checkHeader(data, dataCodec,
- VERSION_START,
- VERSION_CURRENT);
- if (version != version2) {
- throw new CorruptIndexException("Format versions mismatch");
- }
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this.data);
- }
- }
- }
-
- private NumericEntry readNumericEntry(IndexInput meta) {
- NumericEntry entry = new NumericEntry();
- entry.offset = meta.readLong();
- entry.count = meta.readInt();
- entry.missingOffset = meta.readLong();
- if (entry.missingOffset != -1) {
- entry.missingBytes = meta.readLong();
- } else {
- entry.missingBytes = 0;
- }
- entry.byteWidth = meta.readByte();
-
- return entry;
- }
-
- private BinaryEntry readBinaryEntry(IndexInput meta) {
- BinaryEntry entry = new BinaryEntry();
- entry.offset = meta.readLong();
- entry.numBytes = meta.readInt();
- entry.count = meta.readInt();
- entry.missingOffset = meta.readLong();
- if (entry.missingOffset != -1) {
- entry.missingBytes = meta.readLong();
- } else {
- entry.missingBytes = 0;
- }
-
- return entry;
- }
-
- private SortedEntry readSortedEntry(IndexInput meta) {
- SortedEntry entry = new SortedEntry();
- entry.docToOrd = readNumericEntry(meta);
- entry.values = readBinaryEntry(meta);
- return entry;
- }
-
- private SortedSetEntry readSortedSetEntry(IndexInput meta) {
- SortedSetEntry entry = new SortedSetEntry();
- entry.docToOrdAddress = readNumericEntry(meta);
- entry.ords = readNumericEntry(meta);
- entry.values = readBinaryEntry(meta);
- return entry;
- }
-
- private void readFields(IndexInput meta) {
- int fieldNumber = meta.readVInt();
- while (fieldNumber != -1) {
- int fieldType = meta.readByte();
- if (fieldType == NUMBER) {
- numerics.put(fieldNumber, readNumericEntry(meta));
- } else if (fieldType == BYTES) {
- binaries.put(fieldNumber, readBinaryEntry(meta));
- } else if (fieldType == SORTED) {
- sorteds.put(fieldNumber, readSortedEntry(meta));
- } else if (fieldType == SORTED_SET) {
- sortedSets.put(fieldNumber, readSortedSetEntry(meta));
- } else {
- throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
- }
- fieldNumber = meta.readVInt();
- }
- }
-
- @Override
- public long ramBytesUsed() {
- return ramBytesUsed.get();
- }
-
- @Override
- public void checkIntegrity() {
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(data);
- }
- }
-
- @Override
- public synchronized NumericDocValues getNumeric(FieldInfo field) {
- NumericDocValues instance = numericInstances.get(field.number);
- if (instance == null) {
- // Lazy load
- instance = loadNumeric(numerics.get(field.number));
- numericInstances.put(field.number, instance);
- }
- return instance;
- }
-
- private NumericDocValues loadNumeric(NumericEntry entry) {
- data.seek(entry.offset + entry.missingBytes);
- switch (entry.byteWidth) {
- case 1:
- {
- final byte[] values = new byte[entry.count];
- data.readBytes(values, 0, entry.count);
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
- @Override
- public long get(int idx) {
- return values[idx];
- }
- };
- }
-
- case 2:
- {
- final short[] values = new short[entry.count];
- for(int i=0;i<entry.count;i++) {
- values[i] = data.readShort();
- }
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
- @Override
- public long get(int idx) {
- return values[idx];
- }
- };
- }
-
- case 4:
- {
- final int[] values = new int[entry.count];
- for(int i=0;i<entry.count;i++) {
- values[i] = data.readInt();
- }
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
- @Override
- public long get(int idx) {
- return values[idx];
- }
- };
- }
-
- case 8:
- {
- final long[] values = new long[entry.count];
- for(int i=0;i<entry.count;i++) {
- values[i] = data.readLong();
- }
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
- @Override
- public long get(int idx) {
- return values[idx];
- }
- };
- }
-
- default:
- throw new Debug.Assert(ionError();
- }
- }
-
- @Override
- public synchronized BinaryDocValues getBinary(FieldInfo field) {
- BinaryDocValues instance = binaryInstances.get(field.number);
- if (instance == null) {
- // Lazy load
- instance = loadBinary(binaries.get(field.number));
- binaryInstances.put(field.number, instance);
- }
- return instance;
- }
-
- private BinaryDocValues loadBinary(BinaryEntry entry) {
- data.seek(entry.offset);
- final byte[] bytes = new byte[entry.numBytes];
- data.readBytes(bytes, 0, entry.numBytes);
- data.seek(entry.offset + entry.numBytes + entry.missingBytes);
-
- final int[] address = new int[entry.count+1];
- for(int i=0;i<entry.count;i++) {
- address[i] = data.readInt();
- }
- address[entry.count] = data.readInt();
-
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address));
-
- return new BinaryDocValues() {
- @Override
- public void get(int docID, BytesRef result) {
- result.bytes = bytes;
- result.offset = address[docID];
- result.length = address[docID+1] - result.offset;
- };
- };
- }
-
- @Override
- public synchronized SortedDocValues getSorted(FieldInfo field) {
- SortedDocValues instance = sortedInstances.get(field.number);
- if (instance == null) {
- // Lazy load
- instance = loadSorted(field);
- sortedInstances.put(field.number, instance);
- }
- return instance;
- }
-
- private SortedDocValues loadSorted(FieldInfo field) {
- final SortedEntry entry = sorteds.get(field.number);
- final NumericDocValues docToOrd = loadNumeric(entry.docToOrd);
- final BinaryDocValues values = loadBinary(entry.values);
-
- return new SortedDocValues() {
-
- @Override
- public int getOrd(int docID) {
- return (int) docToOrd.get(docID);
- }
-
- @Override
- public void lookupOrd(int ord, BytesRef result) {
- values.get(ord, result);
- }
-
- @Override
- public int getValueCount() {
- return entry.values.count;
- }
-
- // Leave lookupTerm to super's binary search
-
- // Leave termsEnum to super
- };
- }
-
- @Override
- public synchronized SortedSetDocValues getSortedSet(FieldInfo field) {
- SortedSetRawValues instance = sortedSetInstances.get(field.number);
- final SortedSetEntry entry = sortedSets.get(field.number);
- if (instance == null) {
- // Lazy load
- instance = loadSortedSet(entry);
- sortedSetInstances.put(field.number, instance);
- }
-
- final NumericDocValues docToOrdAddress = instance.docToOrdAddress;
- final NumericDocValues ords = instance.ords;
- final BinaryDocValues values = instance.values;
-
- // Must make a new instance since the iterator has state:
- return new RandomAccessOrds() {
- int ordStart;
- int ordUpto;
- int ordLimit;
-
- @Override
- public long nextOrd() {
- if (ordUpto == ordLimit) {
- return NO_MORE_ORDS;
- } else {
- return ords.get(ordUpto++);
- }
- }
-
- @Override
- public void setDocument(int docID) {
- ordStart = ordUpto = (int) docToOrdAddress.get(docID);
- ordLimit = (int) docToOrdAddress.get(docID+1);
- }
-
- @Override
- public void lookupOrd(long ord, BytesRef result) {
- values.get((int) ord, result);
- }
-
- @Override
- public long getValueCount() {
- return entry.values.count;
- }
-
- @Override
- public long ordAt(int index) {
- return ords.get(ordStart + index);
- }
-
- @Override
- public int cardinality() {
- return ordLimit - ordStart;
- }
-
- // Leave lookupTerm to super's binary search
-
- // Leave termsEnum to super
- };
- }
-
- private SortedSetRawValues loadSortedSet(SortedSetEntry entry) {
- SortedSetRawValues instance = new SortedSetRawValues();
- instance.docToOrdAddress = loadNumeric(entry.docToOrdAddress);
- instance.ords = loadNumeric(entry.ords);
- instance.values = loadBinary(entry.values);
- return instance;
- }
-
- private Bits getMissingBits(int fieldNumber, final long offset, final long length) {
- if (offset == -1) {
- return new Bits.MatchAllBits(maxDoc);
- } else {
- Bits instance;
- synchronized(this) {
- instance = docsWithFieldInstances.get(fieldNumber);
- if (instance == null) {
- IndexInput data = this.data.clone();
- data.seek(offset);
- Debug.Assert( length % 8 == 0;
- long bits[] = new long[(int) length >> 3];
- for (int i = 0; i < bits.length; i++) {
- bits[i] = data.readLong();
- }
- instance = new FixedBitSet(bits, maxDoc);
- docsWithFieldInstances.put(fieldNumber, instance);
- }
- }
- return instance;
- }
- }
-
- @Override
- public Bits getDocsWithField(FieldInfo field) {
- switch(field.getDocValuesType()) {
- case SORTED_SET:
- return DocValues.docsWithValue(getSortedSet(field), maxDoc);
- case SORTED:
- return DocValues.docsWithValue(getSorted(field), maxDoc);
- case BINARY:
- BinaryEntry be = binaries.get(field.number);
- return getMissingBits(field.number, be.missingOffset, be.missingBytes);
- case NUMERIC:
- NumericEntry ne = numerics.get(field.number);
- return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
- default:
- throw new Debug.Assert(ionError();
- }
- }
-
- @Override
- public void close() {
- data.close();
- }
-
- static class SortedSetRawValues {
- NumericDocValues docToOrdAddress;
- NumericDocValues ords;
- BinaryDocValues values;
- }
-
- static class NumericEntry {
- long offset;
- int count;
- long missingOffset;
- long missingBytes;
- byte byteWidth;
- int packedIntsVersion;
- }
-
- static class BinaryEntry {
- long offset;
- long missingOffset;
- long missingBytes;
- int count;
- int numBytes;
- int minLength;
- int maxLength;
- int packedIntsVersion;
- int blockSize;
- }
-
- static class SortedEntry {
- NumericEntry docToOrd;
- BinaryEntry values;
- }
-
- static class SortedSetEntry {
- NumericEntry docToOrdAddress;
- NumericEntry ords;
- BinaryEntry values;
- }
-
- static class FSTEntry {
- long offset;
- long numOrds;
- }
-}
+using System.Diagnostics;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using BinaryDocValues = org.apache.lucene.index.BinaryDocValues;
+ using CorruptIndexException = org.apache.lucene.index.CorruptIndexException;
+ using DocValues = org.apache.lucene.index.DocValues;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using NumericDocValues = org.apache.lucene.index.NumericDocValues;
+ using RandomAccessOrds = org.apache.lucene.index.RandomAccessOrds;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using SortedDocValues = org.apache.lucene.index.SortedDocValues;
+ using SortedSetDocValues = org.apache.lucene.index.SortedSetDocValues;
+ using ChecksumIndexInput = org.apache.lucene.store.ChecksumIndexInput;
+ using IndexInput = org.apache.lucene.store.IndexInput;
+ using Bits = org.apache.lucene.util.Bits;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using FixedBitSet = org.apache.lucene.util.FixedBitSet;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using RamUsageEstimator = org.apache.lucene.util.RamUsageEstimator;
+
+ /// <summary>
+ /// Reader for <seealso cref="DirectDocValuesFormat"/>
+ /// </summary>
+
+ internal class DirectDocValuesProducer : DocValuesProducer
+ {
+ // metadata maps (just file pointers and minimal stuff)
+ private readonly IDictionary<int?, NumericEntry> numerics = new Dictionary<int?, NumericEntry>();
+ private readonly IDictionary<int?, BinaryEntry> binaries = new Dictionary<int?, BinaryEntry>();
+ private readonly IDictionary<int?, SortedEntry> sorteds = new Dictionary<int?, SortedEntry>();
+ private readonly IDictionary<int?, SortedSetEntry> sortedSets = new Dictionary<int?, SortedSetEntry>();
+ private readonly IndexInput data;
+
+ // ram instances we have already loaded
+ private readonly IDictionary<int?, NumericDocValues> numericInstances = new Dictionary<int?, NumericDocValues>();
+ private readonly IDictionary<int?, BinaryDocValues> binaryInstances = new Dictionary<int?, BinaryDocValues>();
+ private readonly IDictionary<int?, SortedDocValues> sortedInstances = new Dictionary<int?, SortedDocValues>();
+ private readonly IDictionary<int?, SortedSetRawValues> sortedSetInstances = new Dictionary<int?, SortedSetRawValues>();
+ private readonly IDictionary<int?, Bits> docsWithFieldInstances = new Dictionary<int?, Bits>();
+
+ private readonly int maxDoc;
+ private readonly AtomicLong ramBytesUsed_Renamed;
+ private readonly int version;
+
+ internal const sbyte NUMBER = 0;
+ internal const sbyte BYTES = 1;
+ internal const sbyte SORTED = 2;
+ internal const sbyte SORTED_SET = 3;
+
+ internal const int VERSION_START = 0;
+ internal const int VERSION_CHECKSUM = 1;
+ internal const int VERSION_CURRENT = VERSION_CHECKSUM;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: DirectDocValuesProducer(org.apache.lucene.index.SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws java.io.IOException
+ internal DirectDocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
+ {
+ maxDoc = state.segmentInfo.DocCount;
+ string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+ // read in the entries from the metadata file.
+ ChecksumIndexInput @in = state.directory.openChecksumInput(metaName, state.context);
+ ramBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(this.GetType()));
+ bool success = false;
+ try
+ {
+ version = CodecUtil.checkHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
+ readFields(@in);
+
+ if (version >= VERSION_CHECKSUM)
+ {
+ CodecUtil.checkFooter(@in);
+ }
+ else
+ {
+ CodecUtil.checkEOF(@in);
+ }
+ success = true;
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.close(@in);
+ }
+ else
+ {
+ IOUtils.closeWhileHandlingException(@in);
+ }
+ }
+
+ success = false;
+ try
+ {
+ string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+ data = state.directory.openInput(dataName, state.context);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int version2 = org.apache.lucene.codecs.CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
+ int version2 = CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
+ if (version != version2)
+ {
+ throw new CorruptIndexException("Format versions mismatch");
+ }
+
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(this.data);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private NumericEntry readNumericEntry(org.apache.lucene.store.IndexInput meta) throws java.io.IOException
+ private NumericEntry readNumericEntry(IndexInput meta)
+ {
+ NumericEntry entry = new NumericEntry();
+ entry.offset = meta.readLong();
+ entry.count = meta.readInt();
+ entry.missingOffset = meta.readLong();
+ if (entry.missingOffset != -1)
+ {
+ entry.missingBytes = meta.readLong();
+ }
+ else
+ {
+ entry.missingBytes = 0;
+ }
+ entry.byteWidth = meta.readByte();
+
+ return entry;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private BinaryEntry readBinaryEntry(org.apache.lucene.store.IndexInput meta) throws java.io.IOException
+ private BinaryEntry readBinaryEntry(IndexInput meta)
+ {
+ BinaryEntry entry = new BinaryEntry();
+ entry.offset = meta.readLong();
+ entry.numBytes = meta.readInt();
+ entry.count = meta.readInt();
+ entry.missingOffset = meta.readLong();
+ if (entry.missingOffset != -1)
+ {
+ entry.missingBytes = meta.readLong();
+ }
+ else
+ {
+ entry.missingBytes = 0;
+ }
+
+ return entry;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private SortedEntry readSortedEntry(org.apache.lucene.store.IndexInput meta) throws java.io.IOException
+ private SortedEntry readSortedEntry(IndexInput meta)
+ {
+ SortedEntry entry = new SortedEntry();
+ entry.docToOrd = readNumericEntry(meta);
+ entry.values = readBinaryEntry(meta);
+ return entry;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private SortedSetEntry readSortedSetEntry(org.apache.lucene.store.IndexInput meta) throws java.io.IOException
+ private SortedSetEntry readSortedSetEntry(IndexInput meta)
+ {
+ SortedSetEntry entry = new SortedSetEntry();
+ entry.docToOrdAddress = readNumericEntry(meta);
+ entry.ords = readNumericEntry(meta);
+ entry.values = readBinaryEntry(meta);
+ return entry;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void readFields(org.apache.lucene.store.IndexInput meta) throws java.io.IOException
+ private void readFields(IndexInput meta)
+ {
+ int fieldNumber = meta.readVInt();
+ while (fieldNumber != -1)
+ {
+ int fieldType = meta.readByte();
+ if (fieldType == NUMBER)
+ {
+ numerics[fieldNumber] = readNumericEntry(meta);
+ }
+ else if (fieldType == BYTES)
+ {
+ binaries[fieldNumber] = readBinaryEntry(meta);
+ }
+ else if (fieldType == SORTED)
+ {
+ sorteds[fieldNumber] = readSortedEntry(meta);
+ }
+ else if (fieldType == SORTED_SET)
+ {
+ sortedSets[fieldNumber] = readSortedSetEntry(meta);
+ }
+ else
+ {
+ throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
+ }
+ fieldNumber = meta.readVInt();
+ }
+ }
+
+ public override long ramBytesUsed()
+ {
+ return ramBytesUsed_Renamed.get();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+ public override void checkIntegrity()
+ {
+ if (version >= VERSION_CHECKSUM)
+ {
+ CodecUtil.checksumEntireFile(data);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.NumericDocValues getNumeric(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override NumericDocValues getNumeric(FieldInfo field)
+ {
+ lock (this)
+ {
+ NumericDocValues instance = numericInstances[field.number];
+ if (instance == null)
+ {
+ // Lazy load
+ instance = loadNumeric(numerics[field.number]);
+ numericInstances[field.number] = instance;
+ }
+ return instance;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.NumericDocValues loadNumeric(NumericEntry entry) throws java.io.IOException
+ private NumericDocValues loadNumeric(NumericEntry entry)
+ {
+ data.seek(entry.offset + entry.missingBytes);
+ switch (entry.byteWidth)
+ {
+ case 1:
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final byte[] values = new byte[entry.count];
+ sbyte[] values = new sbyte[entry.count];
+ data.readBytes(values, 0, entry.count);
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(values));
+ return new NumericDocValuesAnonymousInnerClassHelper(this, values);
+ }
+
+ case 2:
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final short[] values = new short[entry.count];
+ short[] values = new short[entry.count];
+ for (int i = 0;i < entry.count;i++)
+ {
+ values[i] = data.readShort();
+ }
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(values));
+ return new NumericDocValuesAnonymousInnerClassHelper2(this, values);
+ }
+
+ case 4:
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int[] values = new int[entry.count];
+ int[] values = new int[entry.count];
+ for (int i = 0;i < entry.count;i++)
+ {
+ values[i] = data.readInt();
+ }
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(values));
+ return new NumericDocValuesAnonymousInnerClassHelper3(this, values);
+ }
+
+ case 8:
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long[] values = new long[entry.count];
+ long[] values = new long[entry.count];
+ for (int i = 0;i < entry.count;i++)
+ {
+ values[i] = data.readLong();
+ }
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(values));
+ return new NumericDocValuesAnonymousInnerClassHelper4(this, values);
+ }
+
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper : NumericDocValues
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private sbyte[] values;
+
+ public NumericDocValuesAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, sbyte[] values)
+ {
+ this.outerInstance = outerInstance;
+ this.values = values;
+ }
+
+ public override long get(int idx)
+ {
+ return values[idx];
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper2 : NumericDocValues
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private short[] values;
+
+ public NumericDocValuesAnonymousInnerClassHelper2(DirectDocValuesProducer outerInstance, short[] values)
+ {
+ this.outerInstance = outerInstance;
+ this.values = values;
+ }
+
+ public override long get(int idx)
+ {
+ return values[idx];
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper3 : NumericDocValues
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private int[] values;
+
+ public NumericDocValuesAnonymousInnerClassHelper3(DirectDocValuesProducer outerInstance, int[] values)
+ {
+ this.outerInstance = outerInstance;
+ this.values = values;
+ }
+
+ public override long get(int idx)
+ {
+ return values[idx];
+ }
+ }
+
+ private class NumericDocValuesAnonymousInnerClassHelper4 : NumericDocValues
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private long[] values;
+
+ public NumericDocValuesAnonymousInnerClassHelper4(DirectDocValuesProducer outerInstance, long[] values)
+ {
+ this.outerInstance = outerInstance;
+ this.values = values;
+ }
+
+ public override long get(int idx)
+ {
+ return values[idx];
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.BinaryDocValues getBinary(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override BinaryDocValues getBinary(FieldInfo field)
+ {
+ lock (this)
+ {
+ BinaryDocValues instance = binaryInstances[field.number];
+ if (instance == null)
+ {
+ // Lazy load
+ instance = loadBinary(binaries[field.number]);
+ binaryInstances[field.number] = instance;
+ }
+ return instance;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.BinaryDocValues loadBinary(BinaryEntry entry) throws java.io.IOException
+ private BinaryDocValues loadBinary(BinaryEntry entry)
+ {
+ data.seek(entry.offset);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final byte[] bytes = new byte[entry.numBytes];
+ sbyte[] bytes = new sbyte[entry.numBytes];
+ data.readBytes(bytes, 0, entry.numBytes);
+ data.seek(entry.offset + entry.numBytes + entry.missingBytes);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int[] address = new int[entry.count+1];
+ int[] address = new int[entry.count + 1];
+ for (int i = 0;i < entry.count;i++)
+ {
+ address[i] = data.readInt();
+ }
+ address[entry.count] = data.readInt();
+
+ ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address));
+
+ return new BinaryDocValuesAnonymousInnerClassHelper(this, bytes, address);
+ }
+
+ private class BinaryDocValuesAnonymousInnerClassHelper : BinaryDocValues
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private sbyte[] bytes;
+ private int[] address;
+
+ public BinaryDocValuesAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, sbyte[] bytes, int[] address)
+ {
+ this.outerInstance = outerInstance;
+ this.bytes = bytes;
+ this.address = address;
+ }
+
+ public override void get(int docID, BytesRef result)
+ {
+ result.bytes = bytes;
+ result.offset = address[docID];
+ result.length = address[docID + 1] - result.offset;
+ };
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.SortedDocValues getSorted(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override SortedDocValues getSorted(FieldInfo field)
+ {
+ lock (this)
+ {
+ SortedDocValues instance = sortedInstances[field.number];
+ if (instance == null)
+ {
+ // Lazy load
+ instance = loadSorted(field);
+ sortedInstances[field.number] = instance;
+ }
+ return instance;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.index.SortedDocValues loadSorted(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ private SortedDocValues loadSorted(FieldInfo field)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final SortedEntry entry = sorteds.get(field.number);
+ SortedEntry entry = sorteds[field.number];
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.NumericDocValues docToOrd = loadNumeric(entry.docToOrd);
+ NumericDocValues docToOrd = loadNumeric(entry.docToOrd);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.BinaryDocValues values = loadBinary(entry.values);
+ BinaryDocValues values = loadBinary(entry.values);
+
+ return new SortedDocValuesAnonymousInnerClassHelper(this, entry, docToOrd, values);
+ }
+
+ private class SortedDocValuesAnonymousInnerClassHelper : SortedDocValues
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private org.apache.lucene.codecs.memory.DirectDocValuesProducer.SortedEntry entry;
+ private NumericDocValues docToOrd;
+ private BinaryDocValues values;
+
+ public SortedDocValuesAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, org.apache.lucene.codecs.memory.DirectDocValuesProducer.SortedEntry entry, NumericDocValues docToOrd, BinaryDocValues values)
+ {
+ this.outerInstance = outerInstance;
+ this.entry = entry;
+ this.docToOrd = docToOrd;
+ this.values = values;
+ }
+
+
+ public override int getOrd(int docID)
+ {
+ return (int) docToOrd.get(docID);
+ }
+
+ public override void lookupOrd(int ord, BytesRef result)
+ {
+ values.get(ord, result);
+ }
+
+ public override int ValueCount
+ {
+ get
+ {
+ return entry.values.count;
+ }
+ }
+
+ // Leave lookupTerm to super's binary search
+
+ // Leave termsEnum to super
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized org.apache.lucene.index.SortedSetDocValues getSortedSet(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override SortedSetDocValues getSortedSet(FieldInfo field)
+ {
+ lock (this)
+ {
+ SortedSetRawValues instance = sortedSetInstances[field.number];
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final SortedSetEntry entry = sortedSets.get(field.number);
+ SortedSetEntry entry = sortedSets[field.number];
+ if (instance == null)
+ {
+ // Lazy load
+ instance = loadSortedSet(entry);
+ sortedSetInstances[field.number] = instance;
+ }
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.NumericDocValues docToOrdAddress = instance.docToOrdAddress;
+ NumericDocValues docToOrdAddress = instance.docToOrdAddress;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.NumericDocValues ords = instance.ords;
+ NumericDocValues ords = instance.ords;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.BinaryDocValues values = instance.values;
+ BinaryDocValues values = instance.values;
+
+ // Must make a new instance since the iterator has state:
+ return new RandomAccessOrdsAnonymousInnerClassHelper(this, entry, docToOrdAddress, ords, values);
+ }
+ }
+
+ private class RandomAccessOrdsAnonymousInnerClassHelper : RandomAccessOrds
+ {
+ private readonly DirectDocValuesProducer outerInstance;
+
+ private org.apache.lucene.codecs.memory.DirectDocValuesProducer.SortedSetEntry entry;
+ private NumericDocValues docToOrdAddress;
+ private NumericDocValues ords;
+ private BinaryDocValues values;
+
+ public RandomAccessOrdsAnonymousInnerClassHelper(DirectDocValuesProducer outerInstance, org.apache.lucene.codecs.memory.DirectDocValuesProducer.SortedSetEntry entry, NumericDocValues docToOrdAddress, NumericDocValues ords, BinaryDocValues values)
+ {
+ this.outerInstance = outerInstance;
+ this.entry = entry;
+ this.docToOrdAddress = docToOrdAddress;
+ this.ords = ords;
+ this.values = values;
+ }
+
+ internal int ordStart;
+ internal int ordUpto;
+ internal int ordLimit;
+
+ public override long nextOrd()
+ {
+ if (ordUpto == ordLimit)
+ {
+ return NO_MORE_ORDS;
+ }
+ else
+ {
+ return ords.get(ordUpto++);
+ }
+ }
+
+ public override int Document
+ {
+ set
+ {
+ ordStart = ordUpto = (int) docToOrdAddress.get(value);
+ ordLimit = (int) docToOrdAddress.get(value+1);
+ }
+ }
+
+ public override void lookupOrd(long ord, BytesRef result)
+ {
+ values.get((int) ord, result);
+ }
+
+ public override long ValueCount
+ {
+ get
+ {
+ return entry.values.count;
+ }
+ }
+
+ public override long ordAt(int index)
+ {
+ return ords.get(ordStart + index);
+ }
+
+ public override int cardinality()
+ {
+ return ordLimit - ordStart;
+ }
+
+ // Leave lookupTerm to super's binary search
+
+ // Leave termsEnum to super
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private SortedSetRawValues loadSortedSet(SortedSetEntry entry) throws java.io.IOException
+ private SortedSetRawValues loadSortedSet(SortedSetEntry entry)
+ {
+ SortedSetRawValues instance = new SortedSetRawValues();
+ instance.docToOrdAddress = loadNumeric(entry.docToOrdAddress);
+ instance.ords = loadNumeric(entry.ords);
+ instance.values = loadBinary(entry.values);
+ return instance;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.Bits getMissingBits(int fieldNumber, final long offset, final long length) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ private Bits getMissingBits(int fieldNumber, long offset, long length)
+ {
+ if (offset == -1)
+ {
+ return new Bits.MatchAllBits(maxDoc);
+ }
+ else
+ {
+ Bits instance;
+ lock (this)
+ {
+ instance = docsWithFieldInstances[fieldNumber];
+ if (instance == null)
+ {
+ IndexInput data = this.data.clone();
+ data.seek(offset);
+ Debug.Assert(length % 8 == 0);
+ long[] bits = new long[(int) length >> 3];
+ for (int i = 0; i < bits.Length; i++)
+ {
+ bits[i] = data.readLong();
+ }
+ instance = new FixedBitSet(bits, maxDoc);
+ docsWithFieldInstances[fieldNumber] = instance;
+ }
+ }
+ return instance;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.Bits getDocsWithField(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override Bits getDocsWithField(FieldInfo field)
+ {
+ switch (field.DocValuesType)
+ {
+ case SORTED_SET:
+ return DocValues.docsWithValue(getSortedSet(field), maxDoc);
+ case SORTED:
+ return DocValues.docsWithValue(getSorted(field), maxDoc);
+ case BINARY:
+ BinaryEntry be = binaries[field.number];
+ return getMissingBits(field.number, be.missingOffset, be.missingBytes);
+ case NUMERIC:
+ NumericEntry ne = numerics[field.number];
+ return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ data.close();
+ }
+
+ internal class SortedSetRawValues
+ {
+ internal NumericDocValues docToOrdAddress;
+ internal NumericDocValues ords;
+ internal BinaryDocValues values;
+ }
+
+ internal class NumericEntry
+ {
+ internal long offset;
+ internal int count;
+ internal long missingOffset;
+ internal long missingBytes;
+ internal sbyte byteWidth;
+ internal int packedIntsVersion;
+ }
+
+ internal class BinaryEntry
+ {
+ internal long offset;
+ internal long missingOffset;
+ internal long missingBytes;
+ internal int count;
+ internal int numBytes;
+ internal int minLength;
+ internal int maxLength;
+ internal int packedIntsVersion;
+ internal int blockSize;
+ }
+
+ internal class SortedEntry
+ {
+ internal NumericEntry docToOrd;
+ internal BinaryEntry values;
+ }
+
+ internal class SortedSetEntry
+ {
+ internal NumericEntry docToOrdAddress;
+ internal NumericEntry ords;
+ internal BinaryEntry values;
+ }
+
+ internal class FSTEntry
+ {
+ internal long offset;
+ internal long numOrds;
+ }
+ }
+
+}
\ No newline at end of file
[06/10] Lucene.Net.Codes/Sep fully ported,
work done on SimpleText and Memory as well
Posted by pn...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
index da1d517..fb81d4d 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
@@ -1,762 +1,1010 @@
-package codecs.memory;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.TreeMap;
-
-import codecs.BlockTermState;
-import codecs.CodecUtil;
-import codecs.FieldsProducer;
-import codecs.PostingsReaderBase;
-import index.CorruptIndexException;
-import index.DocsAndPositionsEnum;
-import index.DocsEnum;
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.SegmentInfo;
-import index.SegmentReadState;
-import index.TermState;
-import index.Terms;
-import index.TermsEnum;
-import store.ByteArrayDataInput;
-import store.IndexInput;
-import util.ArrayUtil;
-import util.Bits;
-import util.BytesRef;
-import util.IOUtils;
-import util.RamUsageEstimator;
-import util.automaton.ByteRunAutomaton;
-import util.automaton.CompiledAutomaton;
-import util.fst.BytesRefFSTEnum.InputOutput;
-import util.fst.BytesRefFSTEnum;
-import util.fst.FST;
-import util.fst.Outputs;
-import util.fst.Util;
-
-/**
- * FST-based terms dictionary reader.
- *
- * The FST directly maps each term and its metadata,
- * it is memory resident.
- *
- * @lucene.experimental
- */
-
-public class FSTTermsReader extends FieldsProducer {
- final TreeMap<String, TermsReader> fields = new TreeMap<>();
- final PostingsReaderBase postingsReader;
- //static bool TEST = false;
- final int version;
-
- public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) {
- final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
-
- this.postingsReader = postingsReader;
- final IndexInput in = state.directory.openInput(termsFileName, state.context);
-
- bool success = false;
- try {
- version = readHeader(in);
- if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
- this.postingsReader.init(in);
- seekDir(in);
-
- final FieldInfos fieldInfos = state.fieldInfos;
- final int numFields = in.readVInt();
- for (int i = 0; i < numFields; i++) {
- int fieldNumber = in.readVInt();
- FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
- long numTerms = in.readVLong();
- long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
- long sumDocFreq = in.readVLong();
- int docCount = in.readVInt();
- int longsSize = in.readVInt();
- TermsReader current = new TermsReader(fieldInfo, in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize);
- TermsReader previous = fields.put(fieldInfo.name, current);
- checkFieldSummary(state.segmentInfo, in, current, previous);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
- }
- }
- }
-
- private int readHeader(IndexInput in) {
- return CodecUtil.checkHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
- FSTTermsWriter.TERMS_VERSION_START,
- FSTTermsWriter.TERMS_VERSION_CURRENT);
- }
- private void seekDir(IndexInput in) {
- if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
- in.seek(in.length() - CodecUtil.footerLength() - 8);
- } else {
- in.seek(in.length() - 8);
- }
- in.seek(in.readLong());
- }
- private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) {
- // #docs with field must be <= #docs
- if (field.docCount < 0 || field.docCount > info.getDocCount()) {
- throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")");
- }
- // #postings must be >= #docs with field
- if (field.sumDocFreq < field.docCount) {
- throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + in + ")");
- }
- // #positions must be >= #postings
- if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) {
- throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + in + ")");
- }
- if (previous != null) {
- throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + in + ")");
- }
- }
-
- @Override
- public Iterator<String> iterator() {
- return Collections.unmodifiableSet(fields.keySet()).iterator();
- }
-
- @Override
- public Terms terms(String field) {
- Debug.Assert( field != null;
- return fields.get(field);
- }
-
- @Override
- public int size() {
- return fields.size();
- }
-
- @Override
- public void close() {
- try {
- IOUtils.close(postingsReader);
- } finally {
- fields.clear();
- }
- }
-
- final class TermsReader extends Terms {
- final FieldInfo fieldInfo;
- final long numTerms;
- final long sumTotalTermFreq;
- final long sumDocFreq;
- final int docCount;
- final int longsSize;
- final FST<FSTTermOutputs.TermData> dict;
-
- TermsReader(FieldInfo fieldInfo, IndexInput in, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) {
- this.fieldInfo = fieldInfo;
- this.numTerms = numTerms;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
- this.longsSize = longsSize;
- this.dict = new FST<>(in, new FSTTermOutputs(fieldInfo, longsSize));
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public bool hasFreqs() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- }
-
- @Override
- public bool hasOffsets() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
-
- @Override
- public bool hasPositions() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
-
- @Override
- public bool hasPayloads() {
- return fieldInfo.hasPayloads();
- }
-
- @Override
- public long size() {
- return numTerms;
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return sumTotalTermFreq;
- }
-
- @Override
- public long getSumDocFreq() {
- return sumDocFreq;
- }
-
- @Override
- public int getDocCount() {
- return docCount;
- }
-
- @Override
- public TermsEnum iterator(TermsEnum reuse) {
- return new SegmentTermsEnum();
- }
-
- @Override
- public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) {
- return new IntersectTermsEnum(compiled, startTerm);
- }
-
- // Only wraps common operations for PBF interact
- abstract class BaseTermsEnum extends TermsEnum {
- /* Current term, null when enum ends or unpositioned */
- BytesRef term;
-
- /* Current term stats + decoded metadata (customized by PBF) */
- final BlockTermState state;
-
- /* Current term stats + undecoded metadata (long[] & byte[]) */
- FSTTermOutputs.TermData meta;
- ByteArrayDataInput bytesReader;
-
- /** Decodes metadata into customized term state */
- abstract void decodeMetaData() ;
-
- BaseTermsEnum() {
- this.state = postingsReader.newTermState();
- this.bytesReader = new ByteArrayDataInput();
- this.term = null;
- // NOTE: metadata will only be initialized in child class
- }
-
- @Override
- public TermState termState() {
- decodeMetaData();
- return state.clone();
- }
-
- @Override
- public BytesRef term() {
- return term;
- }
-
- @Override
- public int docFreq() {
- return state.docFreq;
- }
-
- @Override
- public long totalTermFreq() {
- return state.totalTermFreq;
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
- decodeMetaData();
- return postingsReader.docs(fieldInfo, state, liveDocs, reuse, flags);
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
- if (!hasPositions()) {
- return null;
- }
- decodeMetaData();
- return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags);
- }
-
- @Override
- public void seekExact(long ord) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long ord() {
- throw new UnsupportedOperationException();
- }
- }
-
-
- // Iterates through all terms in this field
- private final class SegmentTermsEnum extends BaseTermsEnum {
- final BytesRefFSTEnum<FSTTermOutputs.TermData> fstEnum;
-
- /* True when current term's metadata is decoded */
- bool decoded;
-
- /* True when current enum is 'positioned' by seekExact(TermState) */
- bool seekPending;
-
- SegmentTermsEnum() {
- super();
- this.fstEnum = new BytesRefFSTEnum<>(dict);
- this.decoded = false;
- this.seekPending = false;
- this.meta = null;
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- // Let PBF decode metadata from long[] and byte[]
- @Override
- void decodeMetaData() {
- if (!decoded && !seekPending) {
- if (meta.bytes != null) {
- bytesReader.reset(meta.bytes, 0, meta.bytes.length);
- }
- postingsReader.decodeTerm(meta.longs, bytesReader, fieldInfo, state, true);
- decoded = true;
- }
- }
-
- // Update current enum according to FSTEnum
- void updateEnum(final InputOutput<FSTTermOutputs.TermData> pair) {
- if (pair == null) {
- term = null;
- } else {
- term = pair.input;
- meta = pair.output;
- state.docFreq = meta.docFreq;
- state.totalTermFreq = meta.totalTermFreq;
- }
- decoded = false;
- seekPending = false;
- }
-
- @Override
- public BytesRef next() {
- if (seekPending) { // previously positioned, but termOutputs not fetched
- seekPending = false;
- SeekStatus status = seekCeil(term);
- Debug.Assert( status == SeekStatus.FOUND; // must positioned on valid term
- }
- updateEnum(fstEnum.next());
- return term;
- }
-
- @Override
- public bool seekExact(BytesRef target) {
- updateEnum(fstEnum.seekExact(target));
- return term != null;
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef target) {
- updateEnum(fstEnum.seekCeil(target));
- if (term == null) {
- return SeekStatus.END;
- } else {
- return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
- }
- }
-
- @Override
- public void seekExact(BytesRef target, TermState otherState) {
- if (!target.equals(term)) {
- state.copyFrom(otherState);
- term = BytesRef.deepCopyOf(target);
- seekPending = true;
- }
- }
- }
-
- // Iterates intersect result with automaton (cannot seek!)
- private final class IntersectTermsEnum extends BaseTermsEnum {
- /* True when current term's metadata is decoded */
- bool decoded;
-
- /* True when there is pending term when calling next() */
- bool pending;
-
- /* stack to record how current term is constructed,
- * used to accumulate metadata or rewind term:
- * level == term.length + 1,
- * == 0 when term is null */
- Frame[] stack;
- int level;
-
- /* to which level the metadata is accumulated
- * so that we can accumulate metadata lazily */
- int metaUpto;
-
- /* term dict fst */
- final FST<FSTTermOutputs.TermData> fst;
- final FST.BytesReader fstReader;
- final Outputs<FSTTermOutputs.TermData> fstOutputs;
-
- /* query automaton to intersect with */
- final ByteRunAutomaton fsa;
-
- private final class Frame {
- /* fst stats */
- FST.Arc<FSTTermOutputs.TermData> fstArc;
-
- /* automaton stats */
- int fsaState;
-
- Frame() {
- this.fstArc = new FST.Arc<>();
- this.fsaState = -1;
- }
-
- public String toString() {
- return "arc=" + fstArc + " state=" + fsaState;
- }
- }
-
- IntersectTermsEnum(CompiledAutomaton compiled, BytesRef startTerm) {
- super();
- //if (TEST) System.out.println("Enum init, startTerm=" + startTerm);
- this.fst = dict;
- this.fstReader = fst.getBytesReader();
- this.fstOutputs = dict.outputs;
- this.fsa = compiled.runAutomaton;
- this.level = -1;
- this.stack = new Frame[16];
- for (int i = 0 ; i < stack.length; i++) {
- this.stack[i] = new Frame();
- }
-
- Frame frame;
- frame = loadVirtualFrame(newFrame());
- this.level++;
- frame = loadFirstFrame(newFrame());
- pushFrame(frame);
-
- this.meta = null;
- this.metaUpto = 1;
- this.decoded = false;
- this.pending = false;
-
- if (startTerm == null) {
- pending = isAccept(topFrame());
- } else {
- doSeekCeil(startTerm);
- pending = !startTerm.equals(term) && isValid(topFrame()) && isAccept(topFrame());
- }
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- void decodeMetaData() {
- Debug.Assert( term != null;
- if (!decoded) {
- if (meta.bytes != null) {
- bytesReader.reset(meta.bytes, 0, meta.bytes.length);
- }
- postingsReader.decodeTerm(meta.longs, bytesReader, fieldInfo, state, true);
- decoded = true;
- }
- }
-
- /** Lazily accumulate meta data, when we got a accepted term */
- void loadMetaData() {
- FST.Arc<FSTTermOutputs.TermData> last, next;
- last = stack[metaUpto].fstArc;
- while (metaUpto != level) {
- metaUpto++;
- next = stack[metaUpto].fstArc;
- next.output = fstOutputs.add(next.output, last.output);
- last = next;
- }
- if (last.isFinal()) {
- meta = fstOutputs.add(last.output, last.nextFinalOutput);
- } else {
- meta = last.output;
- }
- state.docFreq = meta.docFreq;
- state.totalTermFreq = meta.totalTermFreq;
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef target) {
- decoded = false;
- term = doSeekCeil(target);
- loadMetaData();
- if (term == null) {
- return SeekStatus.END;
- } else {
- return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
- }
- }
-
- @Override
- public BytesRef next() {
- //if (TEST) System.out.println("Enum next()");
- if (pending) {
- pending = false;
- loadMetaData();
- return term;
- }
- decoded = false;
- DFS:
- while (level > 0) {
- Frame frame = newFrame();
- if (loadExpandFrame(topFrame(), frame) != null) { // has valid target
- pushFrame(frame);
- if (isAccept(frame)) { // gotcha
- break;
- }
- continue; // check next target
- }
- frame = popFrame();
- while(level > 0) {
- if (loadNextFrame(topFrame(), frame) != null) { // has valid sibling
- pushFrame(frame);
- if (isAccept(frame)) { // gotcha
- break DFS;
- }
- continue DFS; // check next target
- }
- frame = popFrame();
- }
- return null;
- }
- loadMetaData();
- return term;
- }
-
- private BytesRef doSeekCeil(BytesRef target) {
- //if (TEST) System.out.println("Enum doSeekCeil()");
- Frame frame= null;
- int label, upto = 0, limit = target.length;
- while (upto < limit) { // to target prefix, or ceil label (rewind prefix)
- frame = newFrame();
- label = target.bytes[upto] & 0xff;
- frame = loadCeilFrame(label, topFrame(), frame);
- if (frame == null || frame.fstArc.label != label) {
- break;
- }
- Debug.Assert( isValid(frame); // target must be fetched from automaton
- pushFrame(frame);
- upto++;
- }
- if (upto == limit) { // got target
- return term;
- }
- if (frame != null) { // got larger term('s prefix)
- pushFrame(frame);
- return isAccept(frame) ? term : next();
- }
- while (level > 0) { // got target's prefix, advance to larger term
- frame = popFrame();
- while (level > 0 && !canRewind(frame)) {
- frame = popFrame();
- }
- if (loadNextFrame(topFrame(), frame) != null) {
- pushFrame(frame);
- return isAccept(frame) ? term : next();
- }
- }
- return null;
- }
-
- /** Virtual frame, never pop */
- Frame loadVirtualFrame(Frame frame) {
- frame.fstArc.output = fstOutputs.getNoOutput();
- frame.fstArc.nextFinalOutput = fstOutputs.getNoOutput();
- frame.fsaState = -1;
- return frame;
- }
-
- /** Load frame for start arc(node) on fst */
- Frame loadFirstFrame(Frame frame) {
- frame.fstArc = fst.getFirstArc(frame.fstArc);
- frame.fsaState = fsa.getInitialState();
- return frame;
- }
-
- /** Load frame for target arc(node) on fst */
- Frame loadExpandFrame(Frame top, Frame frame) {
- if (!canGrow(top)) {
- return null;
- }
- frame.fstArc = fst.readFirstRealTargetArc(top.fstArc.target, frame.fstArc, fstReader);
- frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label);
- //if (TEST) System.out.println(" loadExpand frame="+frame);
- if (frame.fsaState == -1) {
- return loadNextFrame(top, frame);
- }
- return frame;
- }
-
- /** Load frame for sibling arc(node) on fst */
- Frame loadNextFrame(Frame top, Frame frame) {
- if (!canRewind(frame)) {
- return null;
- }
- while (!frame.fstArc.isLast()) {
- frame.fstArc = fst.readNextRealArc(frame.fstArc, fstReader);
- frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label);
- if (frame.fsaState != -1) {
- break;
- }
- }
- //if (TEST) System.out.println(" loadNext frame="+frame);
- if (frame.fsaState == -1) {
- return null;
- }
- return frame;
- }
-
- /** Load frame for target arc(node) on fst, so that
- * arc.label >= label and !fsa.reject(arc.label) */
- Frame loadCeilFrame(int label, Frame top, Frame frame) {
- FST.Arc<FSTTermOutputs.TermData> arc = frame.fstArc;
- arc = Util.readCeilArc(label, fst, top.fstArc, arc, fstReader);
- if (arc == null) {
- return null;
- }
- frame.fsaState = fsa.step(top.fsaState, arc.label);
- //if (TEST) System.out.println(" loadCeil frame="+frame);
- if (frame.fsaState == -1) {
- return loadNextFrame(top, frame);
- }
- return frame;
- }
-
- bool isAccept(Frame frame) { // reach a term both fst&fsa accepts
- return fsa.isAccept(frame.fsaState) && frame.fstArc.isFinal();
- }
- bool isValid(Frame frame) { // reach a prefix both fst&fsa won't reject
- return /*frame != null &&*/ frame.fsaState != -1;
- }
- bool canGrow(Frame frame) { // can walk forward on both fst&fsa
- return frame.fsaState != -1 && FST.targetHasArcs(frame.fstArc);
- }
- bool canRewind(Frame frame) { // can jump to sibling
- return !frame.fstArc.isLast();
- }
-
- void pushFrame(Frame frame) {
- term = grow(frame.fstArc.label);
- level++;
- //if (TEST) System.out.println(" term=" + term + " level=" + level);
- }
-
- Frame popFrame() {
- term = shrink();
- level--;
- metaUpto = metaUpto > level ? level : metaUpto;
- //if (TEST) System.out.println(" term=" + term + " level=" + level);
- return stack[level+1];
- }
-
- Frame newFrame() {
- if (level+1 == stack.length) {
- final Frame[] temp = new Frame[ArrayUtil.oversize(level+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(stack, 0, temp, 0, stack.length);
- for (int i = stack.length; i < temp.length; i++) {
- temp[i] = new Frame();
- }
- stack = temp;
- }
- return stack[level+1];
- }
-
- Frame topFrame() {
- return stack[level];
- }
-
- BytesRef grow(int label) {
- if (term == null) {
- term = new BytesRef(new byte[16], 0, 0);
- } else {
- if (term.length == term.bytes.length) {
- term.grow(term.length+1);
- }
- term.bytes[term.length++] = (byte)label;
- }
- return term;
- }
-
- BytesRef shrink() {
- if (term.length == 0) {
- term = null;
- } else {
- term.length--;
- }
- return term;
- }
- }
- }
-
- static<T> void walk(FST<T> fst) {
- final ArrayList<FST.Arc<T>> queue = new ArrayList<>();
- final BitSet seen = new BitSet();
- final FST.BytesReader reader = fst.getBytesReader();
- final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());
- queue.add(startArc);
- while (!queue.isEmpty()) {
- final FST.Arc<T> arc = queue.remove(0);
- final long node = arc.target;
- //System.out.println(arc);
- if (FST.targetHasArcs(arc) && !seen.get((int) node)) {
- seen.set((int) node);
- fst.readFirstRealTargetArc(node, arc, reader);
- while (true) {
- queue.add(new FST.Arc<T>().copyFrom(arc));
- if (arc.isLast()) {
- break;
- } else {
- fst.readNextRealArc(arc, reader);
- }
- }
- }
- }
- }
-
- @Override
- public long ramBytesUsed() {
- long ramBytesUsed = 0;
- for (TermsReader r : fields.values()) {
- ramBytesUsed += r.dict == null ? 0 : r.dict.sizeInBytes();
- }
- return ramBytesUsed;
- }
-
- @Override
- public void checkIntegrity() {
- postingsReader.checkIntegrity();
- }
-}
+using System;
+using System.Diagnostics;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace org.apache.lucene.codecs.memory
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using CorruptIndexException = org.apache.lucene.index.CorruptIndexException;
+ using DocsAndPositionsEnum = org.apache.lucene.index.DocsAndPositionsEnum;
+ using DocsEnum = org.apache.lucene.index.DocsEnum;
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using FieldInfos = org.apache.lucene.index.FieldInfos;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentInfo = org.apache.lucene.index.SegmentInfo;
+ using SegmentReadState = org.apache.lucene.index.SegmentReadState;
+ using TermState = org.apache.lucene.index.TermState;
+ using Terms = org.apache.lucene.index.Terms;
+ using TermsEnum = org.apache.lucene.index.TermsEnum;
+ using ByteArrayDataInput = org.apache.lucene.store.ByteArrayDataInput;
+ using IndexInput = org.apache.lucene.store.IndexInput;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using Bits = org.apache.lucene.util.Bits;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using RamUsageEstimator = org.apache.lucene.util.RamUsageEstimator;
+ using ByteRunAutomaton = org.apache.lucene.util.automaton.ByteRunAutomaton;
+ using CompiledAutomaton = org.apache.lucene.util.automaton.CompiledAutomaton;
+ using InputOutput = org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
+ using BytesRefFSTEnum = org.apache.lucene.util.fst.BytesRefFSTEnum;
+ using FST = org.apache.lucene.util.fst.FST;
+ using Outputs = org.apache.lucene.util.fst.Outputs;
+ using Util = org.apache.lucene.util.fst.Util;
+
+ /// <summary>
+ /// FST-based terms dictionary reader.
+ ///
+ /// The FST directly maps each term and its metadata,
+ /// it is memory resident.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+
+ public class FSTTermsReader : FieldsProducer
+ {
+ internal readonly SortedDictionary<string, TermsReader> fields = new SortedDictionary<string, TermsReader>();
+ internal readonly PostingsReaderBase postingsReader;
+ //static boolean TEST = false;
+ internal readonly int version;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public FSTTermsReader(org.apache.lucene.index.SegmentReadState state, org.apache.lucene.codecs.PostingsReaderBase postingsReader) throws java.io.IOException
+ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String termsFileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
+ string termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
+
+ this.postingsReader = postingsReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.store.IndexInput in = state.directory.openInput(termsFileName, state.context);
+ IndexInput @in = state.directory.openInput(termsFileName, state.context);
+
+ bool success = false;
+ try
+ {
+ version = readHeader(@in);
+ if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM)
+ {
+ CodecUtil.checksumEntireFile(@in);
+ }
+ this.postingsReader.init(@in);
+ seekDir(@in);
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.index.FieldInfos fieldInfos = state.fieldInfos;
+ FieldInfos fieldInfos = state.fieldInfos;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int numFields = in.readVInt();
+ int numFields = @in.readVInt();
+ for (int i = 0; i < numFields; i++)
+ {
+ int fieldNumber = @in.readVInt();
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
+ long numTerms = @in.readVLong();
+ long sumTotalTermFreq = fieldInfo.IndexOptions == IndexOptions.DOCS_ONLY ? - 1 : @in.readVLong();
+ long sumDocFreq = @in.readVLong();
+ int docCount = @in.readVInt();
+ int longsSize = @in.readVInt();
+ TermsReader current = new TermsReader(this, fieldInfo, @in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize);
+ TermsReader previous = fields[fieldInfo.name] = current;
+ checkFieldSummary(state.segmentInfo, @in, current, previous);
+ }
+ success = true;
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.close(@in);
+ }
+ else
+ {
+ IOUtils.closeWhileHandlingException(@in);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private int readHeader(org.apache.lucene.store.IndexInput in) throws java.io.IOException
+ private int readHeader(IndexInput @in)
+ {
+ return CodecUtil.checkHeader(@in, FSTTermsWriter.TERMS_CODEC_NAME, FSTTermsWriter.TERMS_VERSION_START, FSTTermsWriter.TERMS_VERSION_CURRENT);
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void seekDir(org.apache.lucene.store.IndexInput in) throws java.io.IOException
+ private void seekDir(IndexInput @in)
+ {
+ if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM)
+ {
+ @in.seek(@in.length() - CodecUtil.footerLength() - 8);
+ }
+ else
+ {
+ @in.seek(@in.length() - 8);
+ }
+ @in.seek(@in.readLong());
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void checkFieldSummary(org.apache.lucene.index.SegmentInfo info, org.apache.lucene.store.IndexInput in, TermsReader field, TermsReader previous) throws java.io.IOException
+ private void checkFieldSummary(SegmentInfo info, IndexInput @in, TermsReader field, TermsReader previous)
+ {
+ // #docs with field must be <= #docs
+ if (field.docCount < 0 || field.docCount > info.DocCount)
+ {
+ throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.DocCount + " (resource=" + @in + ")");
+ }
+ // #postings must be >= #docs with field
+ if (field.sumDocFreq < field.docCount)
+ {
+ throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + @in + ")");
+ }
+ // #positions must be >= #postings
+ if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq)
+ {
+ throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + @in + ")");
+ }
+ if (previous != null)
+ {
+ throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + @in + ")");
+ }
+ }
+
+ public override IEnumerator<string> iterator()
+ {
+ return Collections.unmodifiableSet(fields.Keys).GetEnumerator();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.Terms terms(String field) throws java.io.IOException
+ public override Terms terms(string field)
+ {
+ Debug.Assert(field != null);
+ return fields[field];
+ }
+
+ public override int size()
+ {
+ return fields.Count;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ try
+ {
+ IOUtils.close(postingsReader);
+ }
+ finally
+ {
+ fields.Clear();
+ }
+ }
+
+ internal sealed class TermsReader : Terms
+ {
+ private readonly FSTTermsReader outerInstance;
+
+ internal readonly FieldInfo fieldInfo;
+ internal readonly long numTerms;
+ internal readonly long sumTotalTermFreq;
+ internal readonly long sumDocFreq;
+ internal readonly int docCount;
+ internal readonly int longsSize;
+ internal readonly FST<FSTTermOutputs.TermData> dict;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: TermsReader(org.apache.lucene.index.FieldInfo fieldInfo, org.apache.lucene.store.IndexInput in, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) throws java.io.IOException
+ internal TermsReader(FSTTermsReader outerInstance, FieldInfo fieldInfo, IndexInput @in, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize)
+ {
+ this.outerInstance = outerInstance;
+ this.fieldInfo = fieldInfo;
+ this.numTerms = numTerms;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+ this.longsSize = longsSize;
+ this.dict = new FST<>(@in, new FSTTermOutputs(fieldInfo, longsSize));
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+ public override bool hasFreqs()
+ {
+ return fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ }
+
+ public override bool hasOffsets()
+ {
+ return fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ public override bool hasPositions()
+ {
+ return fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ public override bool hasPayloads()
+ {
+ return fieldInfo.hasPayloads();
+ }
+
+ public override long size()
+ {
+ return numTerms;
+ }
+
+ public override long SumTotalTermFreq
+ {
+ get
+ {
+ return sumTotalTermFreq;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long getSumDocFreq() throws java.io.IOException
+ public override long SumDocFreq
+ {
+ get
+ {
+ return sumDocFreq;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int getDocCount() throws java.io.IOException
+ public override int DocCount
+ {
+ get
+ {
+ return docCount;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.TermsEnum iterator(org.apache.lucene.index.TermsEnum reuse) throws java.io.IOException
+ public override TermsEnum iterator(TermsEnum reuse)
+ {
+ return new SegmentTermsEnum(this);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.TermsEnum intersect(org.apache.lucene.util.automaton.CompiledAutomaton compiled, org.apache.lucene.util.BytesRef startTerm) throws java.io.IOException
+ public override TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm)
+ {
+ return new IntersectTermsEnum(this, compiled, startTerm);
+ }
+
+ // Only wraps common operations for PBF interact
+ internal abstract class BaseTermsEnum : TermsEnum
+ {
+ private readonly FSTTermsReader.TermsReader outerInstance;
+
+ /* Current term, null when enum ends or unpositioned */
+ internal BytesRef term_Renamed;
+
+ /* Current term stats + decoded metadata (customized by PBF) */
+ internal readonly BlockTermState state;
+
+ /* Current term stats + undecoded metadata (long[] & byte[]) */
+ internal FSTTermOutputs.TermData meta;
+ internal ByteArrayDataInput bytesReader;
+
+ /// <summary>
+ /// Decodes metadata into customized term state </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: abstract void decodeMetaData() throws java.io.IOException;
+ internal abstract void decodeMetaData();
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: BaseTermsEnum() throws java.io.IOException
+ internal BaseTermsEnum(FSTTermsReader.TermsReader outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ this.state = outerInstance.outerInstance.postingsReader.newTermState();
+ this.bytesReader = new ByteArrayDataInput();
+ this.term_Renamed = null;
+ // NOTE: metadata will only be initialized in child class
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.TermState termState() throws java.io.IOException
+ public override TermState termState()
+ {
+ decodeMetaData();
+ return state.clone();
+ }
+
+ public override BytesRef term()
+ {
+ return term_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public int docFreq() throws java.io.IOException
+ public override int docFreq()
+ {
+ return state.docFreq;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public long totalTermFreq() throws java.io.IOException
+ public override long totalTermFreq()
+ {
+ return state.totalTermFreq;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsEnum docs(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsEnum reuse, int flags) throws java.io.IOException
+ public override DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
+ {
+ decodeMetaData();
+ return outerInstance.outerInstance.postingsReader.docs(outerInstance.fieldInfo, state, liveDocs, reuse, flags);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.index.DocsAndPositionsEnum docsAndPositions(org.apache.lucene.util.Bits liveDocs, org.apache.lucene.index.DocsAndPositionsEnum reuse, int flags) throws java.io.IOException
+ public override DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags)
+ {
+ if (!outerInstance.hasPositions())
+ {
+ return null;
+ }
+ decodeMetaData();
+ return outerInstance.outerInstance.postingsReader.docsAndPositions(outerInstance.fieldInfo, state, liveDocs, reuse, flags);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void seekExact(long ord) throws java.io.IOException
+ public override void seekExact(long ord)
+ {
+ throw new System.NotSupportedException();
+ }
+
+ public override long ord()
+ {
+ throw new System.NotSupportedException();
+ }
+ }
+
+
+ // Iterates through all terms in this field
+ private sealed class SegmentTermsEnum : BaseTermsEnum
+ {
+ private readonly FSTTermsReader.TermsReader outerInstance;
+
+ internal readonly BytesRefFSTEnum<FSTTermOutputs.TermData> fstEnum;
+
+ /* True when current term's metadata is decoded */
+ internal bool decoded;
+
+ /* True when current enum is 'positioned' by seekExact(TermState) */
+ internal bool seekPending;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: SegmentTermsEnum() throws java.io.IOException
+ internal SegmentTermsEnum(FSTTermsReader.TermsReader outerInstance) : base(outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ this.fstEnum = new BytesRefFSTEnum<>(outerInstance.dict);
+ this.decoded = false;
+ this.seekPending = false;
+ this.meta = null;
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+ // Let PBF decode metadata from long[] and byte[]
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override void decodeMetaData() throws java.io.IOException
+ internal override void decodeMetaData()
+ {
+ if (!decoded && !seekPending)
+ {
+ if (meta.bytes != null)
+ {
+ bytesReader.reset(meta.bytes, 0, meta.bytes.Length);
+ }
+ outerInstance.outerInstance.postingsReader.decodeTerm(meta.longs, bytesReader, outerInstance.fieldInfo, state, true);
+ decoded = true;
+ }
+ }
+
+ // Update current enum according to FSTEnum
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+//ORIGINAL LINE: void updateEnum(final org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput<FSTTermOutputs.TermData> pair)
+ internal void updateEnum(InputOutput<FSTTermOutputs.TermData> pair)
+ {
+ if (pair == null)
+ {
+ term_Renamed = null;
+ }
+ else
+ {
+ term_Renamed = pair.input;
+ meta = pair.output;
+ state.docFreq = meta.docFreq;
+ state.totalTermFreq = meta.totalTermFreq;
+ }
+ decoded = false;
+ seekPending = false;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+ public override BytesRef next()
+ {
+ if (seekPending) // previously positioned, but termOutputs not fetched
+ {
+ seekPending = false;
+ SeekStatus status = seekCeil(term_Renamed);
+ Debug.Assert(status == SeekStatus.FOUND); // must positioned on valid term
+ }
+ updateEnum(fstEnum.next());
+ return term_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean seekExact(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ public override bool seekExact(BytesRef target)
+ {
+ updateEnum(fstEnum.seekExact(target));
+ return term_Renamed != null;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ public override SeekStatus seekCeil(BytesRef target)
+ {
+ updateEnum(fstEnum.seekCeil(target));
+ if (term_Renamed == null)
+ {
+ return SeekStatus.END;
+ }
+ else
+ {
+ return term_Renamed.Equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
+ }
+ }
+
+ public override void seekExact(BytesRef target, TermState otherState)
+ {
+ if (!target.Equals(term_Renamed))
+ {
+ state.copyFrom(otherState);
+ term_Renamed = BytesRef.deepCopyOf(target);
+ seekPending = true;
+ }
+ }
+ }
+
+ // Iterates intersect result with automaton (cannot seek!)
+ private sealed class IntersectTermsEnum : BaseTermsEnum
+ {
+ private readonly FSTTermsReader.TermsReader outerInstance;
+
+ /* True when current term's metadata is decoded */
+ internal bool decoded;
+
+ /* True when there is pending term when calling next() */
+ internal bool pending;
+
+ /* stack to record how current term is constructed,
+ * used to accumulate metadata or rewind term:
+ * level == term.length + 1,
+ * == 0 when term is null */
+ internal Frame[] stack;
+ internal int level;
+
+ /* to which level the metadata is accumulated
+ * so that we can accumulate metadata lazily */
+ internal int metaUpto;
+
+ /* term dict fst */
+ internal readonly FST<FSTTermOutputs.TermData> fst;
+ internal readonly FST.BytesReader fstReader;
+ internal readonly Outputs<FSTTermOutputs.TermData> fstOutputs;
+
+ /* query automaton to intersect with */
+ internal readonly ByteRunAutomaton fsa;
+
+ private sealed class Frame
+ {
+ private readonly FSTTermsReader.TermsReader.IntersectTermsEnum outerInstance;
+
+ /* fst stats */
+ internal FST.Arc<FSTTermOutputs.TermData> fstArc;
+
+ /* automaton stats */
+ internal int fsaState;
+
+ internal Frame(FSTTermsReader.TermsReader.IntersectTermsEnum outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ this.fstArc = new FST.Arc<>();
+ this.fsaState = -1;
+ }
+
+ public override string ToString()
+ {
+ return "arc=" + fstArc + " state=" + fsaState;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: IntersectTermsEnum(org.apache.lucene.util.automaton.CompiledAutomaton compiled, org.apache.lucene.util.BytesRef startTerm) throws java.io.IOException
+ internal IntersectTermsEnum(FSTTermsReader.TermsReader outerInstance, CompiledAutomaton compiled, BytesRef startTerm) : base(outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ //if (TEST) System.out.println("Enum init, startTerm=" + startTerm);
+ this.fst = outerInstance.dict;
+ this.fstReader = fst.BytesReader;
+ this.fstOutputs = outerInstance.dict.outputs;
+ this.fsa = compiled.runAutomaton;
+ this.level = -1;
+ this.stack = new Frame[16];
+ for (int i = 0 ; i < stack.Length; i++)
+ {
+ this.stack[i] = new Frame(this);
+ }
+
+ Frame frame;
+ frame = loadVirtualFrame(newFrame());
+ this.level++;
+ frame = loadFirstFrame(newFrame());
+ pushFrame(frame);
+
+ this.meta = null;
+ this.metaUpto = 1;
+ this.decoded = false;
+ this.pending = false;
+
+ if (startTerm == null)
+ {
+ pending = isAccept(topFrame());
+ }
+ else
+ {
+ doSeekCeil(startTerm);
+ pending = !startTerm.Equals(term_Renamed) && isValid(topFrame()) && isAccept(topFrame());
+ }
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override void decodeMetaData() throws java.io.IOException
+ internal override void decodeMetaData()
+ {
+ Debug.Assert(term_Renamed != null);
+ if (!decoded)
+ {
+ if (meta.bytes != null)
+ {
+ bytesReader.reset(meta.bytes, 0, meta.bytes.Length);
+ }
+ outerInstance.outerInstance.postingsReader.decodeTerm(meta.longs, bytesReader, outerInstance.fieldInfo, state, true);
+ decoded = true;
+ }
+ }
+
+ /// <summary>
+ /// Lazily accumulate meta data, when we got a accepted term </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: void loadMetaData() throws java.io.IOException
+ internal void loadMetaData()
+ {
+ FST.Arc<FSTTermOutputs.TermData> last, next;
+ last = stack[metaUpto].fstArc;
+ while (metaUpto != level)
+ {
+ metaUpto++;
+ next = stack[metaUpto].fstArc;
+ next.output = fstOutputs.add(next.output, last.output);
+ last = next;
+ }
+ if (last.Final)
+ {
+ meta = fstOutputs.add(last.output, last.nextFinalOutput);
+ }
+ else
+ {
+ meta = last.output;
+ }
+ state.docFreq = meta.docFreq;
+ state.totalTermFreq = meta.totalTermFreq;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public SeekStatus seekCeil(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ public override SeekStatus seekCeil(BytesRef target)
+ {
+ decoded = false;
+ term_Renamed = doSeekCeil(target);
+ loadMetaData();
+ if (term_Renamed == null)
+ {
+ return SeekStatus.END;
+ }
+ else
+ {
+ return term_Renamed.Equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.util.BytesRef next() throws java.io.IOException
+ public override BytesRef next()
+ {
+ //if (TEST) System.out.println("Enum next()");
+ if (pending)
+ {
+ pending = false;
+ loadMetaData();
+ return term_Renamed;
+ }
+ decoded = false;
+ while (level > 0)
+ {
+ Frame frame = newFrame();
+ if (loadExpandFrame(topFrame(), frame) != null) // has valid target
+ {
+ pushFrame(frame);
+ if (isAccept(frame)) // gotcha
+ {
+ break;
+ }
+ continue; // check next target
+ }
+ frame = popFrame();
+ while (level > 0)
+ {
+ if (loadNextFrame(topFrame(), frame) != null) // has valid sibling
+ {
+ pushFrame(frame);
+ if (isAccept(frame)) // gotcha
+ {
+ goto DFSBreak;
+ }
+ goto DFSContinue; // check next target
+ }
+ frame = popFrame();
+ }
+ return null;
+ DFSContinue:;
+ }
+ DFSBreak:
+ loadMetaData();
+ return term_Renamed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private org.apache.lucene.util.BytesRef doSeekCeil(org.apache.lucene.util.BytesRef target) throws java.io.IOException
+ internal BytesRef doSeekCeil(BytesRef target)
+ {
+ //if (TEST) System.out.println("Enum doSeekCeil()");
+ Frame frame = null;
+ int label , upto = 0, limit = target.length;
+ while (upto < limit) // to target prefix, or ceil label (rewind prefix)
+ {
+ frame = newFrame();
+ label = target.bytes[upto] & 0xff;
+ frame = loadCeilFrame(label, topFrame(), frame);
+ if (frame == null || frame.fstArc.label != label)
+ {
+ break;
+ }
+ Debug.Assert(isValid(frame)); // target must be fetched from automaton
+ pushFrame(frame);
+ upto++;
+ }
+ if (upto == limit) // got target
+ {
+ return term_Renamed;
+ }
+ if (frame != null) // got larger term('s prefix)
+ {
+ pushFrame(frame);
+ return isAccept(frame) ? term_Renamed : next();
+ }
+ while (level > 0) // got target's prefix, advance to larger term
+ {
+ frame = popFrame();
+ while (level > 0 && !canRewind(frame))
+ {
+ frame = popFrame();
+ }
+ if (loadNextFrame(topFrame(), frame) != null)
+ {
+ pushFrame(frame);
+ return isAccept(frame) ? term_Renamed : next();
+ }
+ }
+ return null;
+ }
+
+ /// <summary>
+ /// Virtual frame, never pop </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadVirtualFrame(Frame frame) throws java.io.IOException
+ internal Frame loadVirtualFrame(Frame frame)
+ {
+ frame.fstArc.output = fstOutputs.NoOutput;
+ frame.fstArc.nextFinalOutput = fstOutputs.NoOutput;
+ frame.fsaState = -1;
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for start arc(node) on fst </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadFirstFrame(Frame frame) throws java.io.IOException
+ internal Frame loadFirstFrame(Frame frame)
+ {
+ frame.fstArc = fst.getFirstArc(frame.fstArc);
+ frame.fsaState = fsa.InitialState;
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for target arc(node) on fst </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadExpandFrame(Frame top, Frame frame) throws java.io.IOException
+ internal Frame loadExpandFrame(Frame top, Frame frame)
+ {
+ if (!canGrow(top))
+ {
+ return null;
+ }
+ frame.fstArc = fst.readFirstRealTargetArc(top.fstArc.target, frame.fstArc, fstReader);
+ frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label);
+ //if (TEST) System.out.println(" loadExpand frame="+frame);
+ if (frame.fsaState == -1)
+ {
+ return loadNextFrame(top, frame);
+ }
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for sibling arc(node) on fst </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadNextFrame(Frame top, Frame frame) throws java.io.IOException
+ internal Frame loadNextFrame(Frame top, Frame frame)
+ {
+ if (!canRewind(frame))
+ {
+ return null;
+ }
+ while (!frame.fstArc.Last)
+ {
+ frame.fstArc = fst.readNextRealArc(frame.fstArc, fstReader);
+ frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label);
+ if (frame.fsaState != -1)
+ {
+ break;
+ }
+ }
+ //if (TEST) System.out.println(" loadNext frame="+frame);
+ if (frame.fsaState == -1)
+ {
+ return null;
+ }
+ return frame;
+ }
+
+ /// <summary>
+ /// Load frame for target arc(node) on fst, so that
+ /// arc.label >= label and !fsa.reject(arc.label)
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: Frame loadCeilFrame(int label, Frame top, Frame frame) throws java.io.IOException
+ internal Frame loadCeilFrame(int label, Frame top, Frame frame)
+ {
+ FST.Arc<FSTTermOutputs.TermData> arc = frame.fstArc;
+ arc = Util.readCeilArc(label, fst, top.fstArc, arc, fstReader);
+ if (arc == null)
+ {
+ return null;
+ }
+ frame.fsaState = fsa.step(top.fsaState, arc.label);
+ //if (TEST) System.out.println(" loadCeil frame="+frame);
+ if (frame.fsaState == -1)
+ {
+ return loadNextFrame(top, frame);
+ }
+ return frame;
+ }
+
+ internal bool isAccept(Frame frame) // reach a term both fst&fsa accepts
+ {
+ return fsa.isAccept(frame.fsaState) && frame.fstArc.Final;
+ }
+ internal bool isValid(Frame frame) // reach a prefix both fst&fsa won't reject
+ {
+ return frame.fsaState != -1; //frame != null &&
+ }
+ internal bool canGrow(Frame frame) // can walk forward on both fst&fsa
+ {
+ return frame.fsaState != -1 && FST.targetHasArcs(frame.fstArc);
+ }
+ internal bool canRewind(Frame frame) // can jump to sibling
+ {
+ return !frame.fstArc.Last;
+ }
+
+ internal void pushFrame(Frame frame)
+ {
+ term_Renamed = grow(frame.fstArc.label);
+ level++;
+ //if (TEST) System.out.println(" term=" + term + " level=" + level);
+ }
+
+ internal Frame popFrame()
+ {
+ term_Renamed = shrink();
+ level--;
+ metaUpto = metaUpto > level ? level : metaUpto;
+ //if (TEST) System.out.println(" term=" + term + " level=" + level);
+ return stack[level + 1];
+ }
+
+ internal Frame newFrame()
+ {
+ if (level + 1 == stack.Length)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Frame[] temp = new Frame[org.apache.lucene.util.ArrayUtil.oversize(level+2, org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ Frame[] temp = new Frame[ArrayUtil.oversize(level + 2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ Array.Copy(stack, 0, temp, 0, stack.Length);
+ for (int i = stack.Length; i < temp.Length; i++)
+ {
+ temp[i] = new Frame(this);
+ }
+ stack = temp;
+ }
+ return stack[level + 1];
+ }
+
+ internal Frame topFrame()
+ {
+ return stack[level];
+ }
+
+ internal BytesRef grow(int label)
+ {
+ if (term_Renamed == null)
+ {
+ term_Renamed = new BytesRef(new sbyte[16], 0, 0);
+ }
+ else
+ {
+ if (term_Renamed.length == term_Renamed.bytes.length)
+ {
+ term_Renamed.grow(term_Renamed.length + 1);
+ }
+ term_Renamed.bytes[term_Renamed.length++] = (sbyte)label;
+ }
+ return term_Renamed;
+ }
+
+ internal BytesRef shrink()
+ {
+ if (term_Renamed.length == 0)
+ {
+ term_Renamed = null;
+ }
+ else
+ {
+ term_Renamed.length--;
+ }
+ return term_Renamed;
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: static<T> void walk(org.apache.lucene.util.fst.FST<T> fst) throws java.io.IOException
+ internal static void walk<T>(FST<T> fst)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.ArrayList<org.apache.lucene.util.fst.FST.Arc<T>> queue = new java.util.ArrayList<>();
+ List<FST.Arc<T>> queue = new List<FST.Arc<T>>();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.util.BitSet seen = new java.util.BitSet();
+ BitArray seen = new BitArray();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.BytesReader reader = fst.getBytesReader();
+ FST.BytesReader reader = fst.BytesReader;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<T> startArc = fst.getFirstArc(new org.apache.lucene.util.fst.FST.Arc<T>());
+ FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());
+ queue.Add(startArc);
+ while (queue.Count > 0)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST.Arc<T> arc = queue.remove(0);
+ FST.Arc<T> arc = queue.Remove(0);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long node = arc.target;
+ long node = arc.target;
+ //System.out.println(arc);
+ if (FST.targetHasArcs(arc) && !seen.Get((int) node))
+ {
+ seen.Set((int) node, true);
+ fst.readFirstRealTargetArc(node, arc, reader);
+ while (true)
+ {
+ queue.Add((new FST.Arc<T>()).copyFrom(arc));
+ if (arc.Last)
+ {
+ break;
+ }
+ else
+ {
+ fst.readNextRealArc(arc, reader);
+ }
+ }
+ }
+ }
+ }
+
+ public override long ramBytesUsed()
+ {
+ long ramBytesUsed = 0;
+ foreach (TermsReader r in fields.Values)
+ {
+ ramBytesUsed += r.dict == null ? 0 : r.dict.sizeInBytes();
+ }
+ return ramBytesUsed;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException
+ public override void checkIntegrity()
+ {
+ postingsReader.checkIntegrity();
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/d852d5b0/src/Lucene.Net.Codecs/Memory/FSTTermsWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermsWriter.cs b/src/Lucene.Net.Codecs/Memory/FSTTermsWriter.cs
index 4b587b7..785b137 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTTermsWriter.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTTermsWriter.cs
@@ -1,277 +1,329 @@
-package codecs.memory;
+using System.Collections.Generic;
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+namespace org.apache.lucene.codecs.memory
+{
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Comparator;
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import index.FieldInfo.IndexOptions;
-import index.FieldInfo;
-import index.FieldInfos;
-import index.IndexFileNames;
-import index.SegmentWriteState;
-import store.DataOutput;
-import store.IndexOutput;
-import store.RAMOutputStream;
-import util.ArrayUtil;
-import util.BytesRef;
-import util.IOUtils;
-import util.IntsRef;
-import util.fst.Builder;
-import util.fst.FST;
-import util.fst.Util;
-import codecs.BlockTermState;
-import codecs.PostingsWriterBase;
-import codecs.PostingsConsumer;
-import codecs.FieldsConsumer;
-import codecs.TermsConsumer;
-import codecs.TermStats;
-import codecs.CodecUtil;
-/**
- * FST-based term dict, using metadata as FST output.
- *
- * The FST directly holds the mapping between <term, metadata>.
- *
- * Term metadata consists of three parts:
- * 1. term statistics: docFreq, totalTermFreq;
- * 2. monotonic long[], e.g. the pointer to the postings list for that term;
- * 3. generic byte[], e.g. other information need by postings reader.
- *
- * <p>
- * File:
- * <ul>
- * <li><tt>.tst</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
- * </ul>
- * <p>
- *
- * <a name="Termdictionary" id="Termdictionary"></a>
- * <h3>Term Dictionary</h3>
- * <p>
- * The .tst contains a list of FSTs, one for each field.
- * The FST maps a term to its corresponding statistics (e.g. docfreq)
- * and metadata (e.g. information for postings list reader like file pointer
- * to postings list).
- * </p>
- * <p>
- * Typically the metadata is separated into two parts:
- * <ul>
- * <li>
- * Monotonical long array: Some metadata will always be ascending in order
- * with the corresponding term. This part is used by FST to share outputs between arcs.
- * </li>
- * <li>
- * Generic byte array: Used to store non-monotonic metadata.
- * </li>
- * </ul>
- * </p>
- *
- * File format:
- * <ul>
- * <li>TermsDict(.tst) --> Header, <i>PostingsHeader</i>, FieldSummary, DirOffset</li>
- * <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, SumTotalTermFreq?,
- * SumDocFreq, DocCount, LongsSize, TermFST ><sup>NumFields</sup></li>
- * <li>TermFST --> {@link FST FST<TermData>}</li>
- * <li>TermData --> Flag, BytesSize?, LongDelta<sup>LongsSize</sup>?, Byte<sup>BytesSize</sup>?,
- * < DocFreq[Same?], (TotalTermFreq-DocFreq) > ? </li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
- * <li>DocFreq, LongsSize, BytesSize, NumFields,
- * FieldNumber, DocCount --> {@link DataOutput#writeVInt VInt}</li>
- * <li>TotalTermFreq, NumTerms, SumTotalTermFreq, SumDocFreq, LongDelta -->
- * {@link DataOutput#writeVLong VLong}</li>
- * </ul>
- * <p>Notes:</p>
- * <ul>
- * <li>
- * The format of PostingsHeader and generic meta bytes are customized by the specific postings implementation:
- * they contain arbitrary per-file data (such as parameters or versioning information), and per-term data
- * (non-monotonic ones like pulsed postings data).
- * </li>
- * <li>
- * The format of TermData is determined by FST, typically monotonic metadata will be dense around shallow arcs,
- * while in deeper arcs only generic bytes and term statistics exist.
- * </li>
- * <li>
- * The byte Flag is used to indicate which part of metadata exists on current arc. Specially the monotonic part
- * is omitted when it is an array of 0s.
- * </li>
- * <li>
- * Since LongsSize is per-field fixed, it is only written once in field summary.
- * </li>
- * </ul>
- *
- * @lucene.experimental
- */
+ using IndexOptions = org.apache.lucene.index.FieldInfo.IndexOptions;
+ using FieldInfo = org.apache.lucene.index.FieldInfo;
+ using FieldInfos = org.apache.lucene.index.FieldInfos;
+ using IndexFileNames = org.apache.lucene.index.IndexFileNames;
+ using SegmentWriteState = org.apache.lucene.index.SegmentWriteState;
+ using DataOutput = org.apache.lucene.store.DataOutput;
+ using IndexOutput = org.apache.lucene.store.IndexOutput;
+ using RAMOutputStream = org.apache.lucene.store.RAMOutputStream;
+ using ArrayUtil = org.apache.lucene.util.ArrayUtil;
+ using BytesRef = org.apache.lucene.util.BytesRef;
+ using IOUtils = org.apache.lucene.util.IOUtils;
+ using IntsRef = org.apache.lucene.util.IntsRef;
+ using Builder = org.apache.lucene.util.fst.Builder;
+ using FST = org.apache.lucene.util.fst.FST;
+ using Util = org.apache.lucene.util.fst.Util;
-public class FSTTermsWriter extends FieldsConsumer {
- static final String TERMS_EXTENSION = "tmp";
- static final String TERMS_CODEC_NAME = "FST_TERMS_DICT";
- public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CHECKSUM = 1;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
-
- final PostingsWriterBase postingsWriter;
- final FieldInfos fieldInfos;
- IndexOutput out;
- final List<FieldMetaData> fields = new ArrayList<>();
+ /// <summary>
+ /// FST-based term dict, using metadata as FST output.
+ ///
+ /// The FST directly holds the mapping between <term, metadata>.
+ ///
+ /// Term metadata consists of three parts:
+ /// 1. term statistics: docFreq, totalTermFreq;
+ /// 2. monotonic long[], e.g. the pointer to the postings list for that term;
+ /// 3. generic byte[], e.g. other information need by postings reader.
+ ///
+ /// <para>
+ /// File:
+ /// <ul>
+ /// <li><tt>.tst</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
+ /// </ul>
+ /// </para>
+ /// <para>
+ ///
+ /// <a name="Termdictionary" id="Termdictionary"></a>
+ /// <h3>Term Dictionary</h3>
+ /// </para>
+ /// <para>
+ /// The .tst contains a list of FSTs, one for each field.
+ /// The FST maps a term to its corresponding statistics (e.g. docfreq)
+ /// and metadata (e.g. information for postings list reader like file pointer
+ /// to postings list).
+ /// </para>
+ /// <para>
+ /// Typically the metadata is separated into two parts:
+ /// <ul>
+ /// <li>
+ /// Monotonical long array: Some metadata will always be ascending in order
+ /// with the corresponding term. This part is used by FST to share outputs between arcs.
+ /// </li>
+ /// <li>
+ /// Generic byte array: Used to store non-monotonic metadata.
+ /// </li>
+ /// </ul>
+ /// </para>
+ ///
+ /// File format:
+ /// <ul>
+ /// <li>TermsDict(.tst) --> Header, <i>PostingsHeader</i>, FieldSummary, DirOffset</li>
+ /// <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, SumTotalTermFreq?,
+ /// SumDocFreq, DocCount, LongsSize, TermFST ><sup>NumFields</sup></li>
+ /// <li>TermFST --> <seealso cref="FST FST<TermData>"/></li>
+ /// <li>TermData --> Flag, BytesSize?, LongDelta<sup>LongsSize</sup>?, Byte<sup>BytesSize</sup>?,
+ /// < DocFreq[Same?], (TotalTermFreq-DocFreq) > ? </li>
+ /// <li>Header --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
+ /// <li>DirOffset --> <seealso cref="DataOutput#writeLong Uint64"/></li>
+ /// <li>DocFreq, LongsSize, BytesSize, NumFields,
+ /// FieldNumber, DocCount --> <seealso cref="DataOutput#writeVInt VInt"/></li>
+ /// <li>TotalTermFreq, NumTerms, SumTotalTermFreq, SumDocFreq, LongDelta -->
+ /// <seealso cref="DataOutput#writeVLong VLong"/></li>
+ /// </ul>
+ /// <para>Notes:</para>
+ /// <ul>
+ /// <li>
+ /// The format of PostingsHeader and generic meta bytes are customized by the specific postings implementation:
+ /// they contain arbitrary per-file data (such as parameters or versioning information), and per-term data
+ /// (non-monotonic ones like pulsed postings data).
+ /// </li>
+ /// <li>
+ /// The format of TermData is determined by FST, typically monotonic metadata will be dense around shallow arcs,
+ /// while in deeper arcs only generic bytes and term statistics exist.
+ /// </li>
+ /// <li>
+ /// The byte Flag is used to indicate which part of metadata exists on current arc. Specially the monotonic part
+ /// is omitted when it is an array of 0s.
+ /// </li>
+ /// <li>
+ /// Since LongsSize is per-field fixed, it is only written once in field summary.
+ /// </li>
+ /// </ul>
+ ///
+ /// @lucene.experimental
+ /// </summary>
- public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) {
- final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION);
+ public class FSTTermsWriter : FieldsConsumer
+ {
+ internal const string TERMS_EXTENSION = "tmp";
+ internal const string TERMS_CODEC_NAME = "FST_TERMS_DICT";
+ public const int TERMS_VERSION_START = 0;
+ public const int TERMS_VERSION_CHECKSUM = 1;
+ public const int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
- this.postingsWriter = postingsWriter;
- this.fieldInfos = state.fieldInfos;
- this.out = state.directory.createOutput(termsFileName, state.context);
+ internal readonly PostingsWriterBase postingsWriter;
+ internal readonly FieldInfos fieldInfos;
+ internal IndexOutput @out;
+ internal readonly IList<FieldMetaData> fields = new List<FieldMetaData>();
- bool success = false;
- try {
- writeHeader(out);
- this.postingsWriter.init(out);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(out);
- }
- }
- }
- private void writeHeader(IndexOutput out) {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- }
- private void writeTrailer(IndexOutput out, long dirStart) {
- out.writeLong(dirStart);
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public FSTTermsWriter(org.apache.lucene.index.SegmentWriteState state, org.apache.lucene.codecs.PostingsWriterBase postingsWriter) throws java.io.IOException
+ public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String termsFileName = org.apache.lucene.index.IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION);
+ string termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION);
- @Override
- public TermsConsumer addField(FieldInfo field) {
- return new TermsWriter(field);
- }
+ this.postingsWriter = postingsWriter;
+ this.fieldInfos = state.fieldInfos;
+ this.@out = state.directory.createOutput(termsFileName, state.context);
- @Override
- public void close() {
- if (out != null) {
- IOException ioe = null;
- try {
- // write field summary
- final long dirStart = out.getFilePointer();
-
- out.writeVInt(fields.size());
- for (FieldMetaData field : fields) {
- out.writeVInt(field.fieldInfo.number);
- out.writeVLong(field.numTerms);
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- out.writeVLong(field.sumTotalTermFreq);
- }
- out.writeVLong(field.sumDocFreq);
- out.writeVInt(field.docCount);
- out.writeVInt(field.longsSize);
- field.dict.save(out);
- }
- writeTrailer(out, dirStart);
- CodecUtil.writeFooter(out);
- } catch (IOException ioe2) {
- ioe = ioe2;
- } finally {
- IOUtils.closeWhileHandlingException(ioe, out, postingsWriter);
- out = null;
- }
- }
- }
+ bool success = false;
+ try
+ {
+ writeHeader(@out);
+ this.postingsWriter.init(@out);
+ success = true;
+ }
+ finally
+ {
+ if (!success)
+ {
+ IOUtils.closeWhileHandlingException(@out);
+ }
+ }
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void writeHeader(org.apache.lucene.store.IndexOutput out) throws java.io.IOException
+ private void writeHeader(IndexOutput @out)
+ {
+ CodecUtil.writeHeader(@out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void writeTrailer(org.apache.lucene.store.IndexOutput out, long dirStart) throws java.io.IOException
+ private void writeTrailer(IndexOutput @out, long dirStart)
+ {
+ @out.writeLong(dirStart);
+ }
- private static class FieldMetaData {
- public final FieldInfo fieldInfo;
- public final long numTerms;
- public final long sumTotalTermFreq;
- public final long sumDocFreq;
- public final int docCount;
- public final int longsSize;
- public final FST<FSTTermOutputs.TermData> dict;
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.TermsConsumer addField(org.apache.lucene.index.FieldInfo field) throws java.io.IOException
+ public override TermsConsumer addField(FieldInfo field)
+ {
+ return new TermsWriter(this, field);
+ }
- public FieldMetaData(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST<FSTTermOutputs.TermData> fst) {
- this.fieldInfo = fieldInfo;
- this.numTerms = numTerms;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
- this.longsSize = longsSize;
- this.dict = fst;
- }
- }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void close() throws java.io.IOException
+ public override void close()
+ {
+ if (@out != null)
+ {
+ IOException ioe = null;
+ try
+ {
+ // write field summary
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long dirStart = out.getFilePointer();
+ long dirStart = @out.FilePointer;
- final class TermsWriter extends TermsConsumer {
- private final Builder<FSTTermOutputs.TermData> builder;
- private final FSTTermOutputs outputs;
- private final FieldInfo fieldInfo;
- private final int longsSize;
- private long numTerms;
+ @out.writeVInt(fields.Count);
+ foreach (FieldMetaData field in fields)
+ {
+ @out.writeVInt(field.fieldInfo.number);
+ @out.writeVLong(field.numTerms);
+ if (field.fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY)
+ {
+ @out.writeVLong(field.sumTotalTermFreq);
+ }
+ @out.writeVLong(field.sumDocFreq);
+ @out.writeVInt(field.docCount);
+ @out.writeVInt(field.longsSize);
+ field.dict.save(@out);
+ }
+ writeTrailer(@out, dirStart);
+ CodecUtil.writeFooter(@out);
+ }
+ catch (IOException ioe2)
+ {
+ ioe = ioe2;
+ }
+ finally
+ {
+ IOUtils.closeWhileHandlingException(ioe, @out, postingsWriter);
+ @out = null;
+ }
+ }
+ }
- private final IntsRef scratchTerm = new IntsRef();
- private final RAMOutputStream statsWriter = new RAMOutputStream();
- private final RAMOutputStream metaWriter = new RAMOutputStream();
+ private class FieldMetaData
+ {
+ public readonly FieldInfo fieldInfo;
+ public readonly long numTerms;
+ public readonly long sumTotalTermFreq;
+ public readonly long sumDocFreq;
+ public readonly int docCount;
+ public readonly int longsSize;
+ public readonly FST<FSTTermOutputs.TermData> dict;
- TermsWriter(FieldInfo fieldInfo) {
- this.numTerms = 0;
- this.fieldInfo = fieldInfo;
- this.longsSize = postingsWriter.setField(fieldInfo);
- this.outputs = new FSTTermOutputs(fieldInfo, longsSize);
- this.builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
- }
+ public FieldMetaData(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST<FSTTermOutputs.TermData> fst)
+ {
+ this.fieldInfo = fieldInfo;
+ this.numTerms = numTerms;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+ this.longsSize = longsSize;
+ this.dict = fst;
+ }
+ }
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
+ internal sealed class TermsWriter : TermsConsumer
+ {
+ private readonly FSTTermsWriter outerInstance;
- @Override
- public PostingsConsumer startTerm(BytesRef text) {
- postingsWriter.startTerm();
- return postingsWriter;
- }
+ internal readonly Builder<FSTTermOutputs.TermData> builder;
+ internal readonly FSTTermOutputs outputs;
+ internal readonly FieldInfo fieldInfo;
+ internal readonly int longsSize;
+ internal long numTerms;
- @Override
- public void finishTerm(BytesRef text, TermStats stats) {
- // write term meta data into fst
- final BlockTermState state = postingsWriter.newTermState();
- final FSTTermOutputs.TermData meta = new FSTTermOutputs.TermData();
- meta.longs = new long[longsSize];
- meta.bytes = null;
- meta.docFreq = state.docFreq = stats.docFreq;
- meta.totalTermFreq = state.totalTermFreq = stats.totalTermFreq;
- postingsWriter.finishTerm(state);
- postingsWriter.encodeTerm(meta.longs, metaWriter, fieldInfo, state, true);
- final int bytesSize = (int)metaWriter.getFilePointer();
- if (bytesSize > 0) {
- meta.bytes = new byte[bytesSize];
- metaWriter.writeTo(meta.bytes, 0);
- metaWriter.reset();
- }
- builder.add(Util.toIntsRef(text, scratchTerm), meta);
- numTerms++;
- }
+ internal readonly IntsRef scratchTerm = new IntsRef();
+ internal readonly RAMOutputStream statsWriter = new RAMOutputStream();
+ internal readonly RAMOutputStream metaWriter = new RAMOutputStream();
- @Override
- public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) {
- // save FST dict
- if (numTerms > 0) {
- final FST<FSTTermOutputs.TermData> fst = builder.finish();
- fields.add(new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, fst));
- }
- }
- }
-}
+ internal TermsWriter(FSTTermsWriter outerInstance, FieldInfo fieldInfo)
+ {
+ this.outerInstance = outerInstance;
+ this.numTerms = 0;
+ this.fieldInfo = fieldInfo;
+ this.longsSize = outerInstance.postingsWriter.setField(fieldInfo);
+ this.outputs = new FSTTermOutputs(fieldInfo, longsSize);
+ this.builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
+ }
+
+ public override IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return BytesRef.UTF8SortedAsUnicodeComparator;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public org.apache.lucene.codecs.PostingsConsumer startTerm(org.apache.lucene.util.BytesRef text) throws java.io.IOException
+ public override PostingsConsumer startTerm(BytesRef text)
+ {
+ outerInstance.postingsWriter.startTerm();
+ return outerInstance.postingsWriter;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void finishTerm(org.apache.lucene.util.BytesRef text, org.apache.lucene.codecs.TermStats stats) throws java.io.IOException
+ public override void finishTerm(BytesRef text, TermStats stats)
+ {
+ // write term meta data into fst
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.codecs.BlockTermState state = postingsWriter.newTermState();
+ BlockTermState state = outerInstance.postingsWriter.newTermState();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final FSTTermOutputs.TermData meta = new FSTTermOutputs.TermData();
+ FSTTermOutputs.TermData meta = new FSTTermOutputs.TermData();
+ meta.longs = new long[longsSize];
+ meta.bytes = null;
+ meta.docFreq = state.docFreq = stats.docFreq;
+ meta.totalTermFreq = state.totalTermFreq = stats.totalTermFreq;
+ outerInstance.postingsWriter.finishTerm(state);
+ outerInstance.postingsWriter.encodeTerm(meta.longs, metaWriter, fieldInfo, state, true);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int bytesSize = (int)metaWriter.getFilePointer();
+ int bytesSize = (int)metaWriter.FilePointer;
+ if (bytesSize > 0)
+ {
+ meta.bytes = new sbyte[bytesSize];
+ metaWriter.writeTo(meta.bytes, 0);
+ metaWriter.reset();
+ }
+ builder.add(Util.toIntsRef(text, scratchTerm), meta);
+ numTerms++;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws java.io.IOException
+ public override void finish(long sumTotalTermFreq, long sumDocFreq, int docCount)
+ {
+ // save FST dict
+ if (numTerms > 0)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.fst.FST<FSTTermOutputs.TermData> fst = builder.finish();
+ FST<FSTTermOutputs.TermData> fst = builder.finish();
+ outerInstance.fields.Add(new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, fst));
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file