You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/01/29 13:27:55 UTC
[33/37] lucenenet git commit: Lucene.Net.Codecs: Renamed Intblock
folder to IntBlock
Lucene.Net.Codecs: Renamed Intblock folder to IntBlock
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/0159ad2f
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/0159ad2f
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/0159ad2f
Branch: refs/heads/api-work
Commit: 0159ad2fcfc39dc4e662326ca01c414f0350fdae
Parents: 5a5c215
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Sun Jan 29 18:00:51 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Sun Jan 29 18:00:51 2017 +0700
----------------------------------------------------------------------
.../IntBlock/FixedIntBlockIndexInput.cs | 196 ++++++++++++++++
.../IntBlock/FixedIntBlockIndexOutput.cs | 151 ++++++++++++
.../IntBlock/VariableIntBlockIndexInput.cs | 229 +++++++++++++++++++
.../IntBlock/VariableIntBlockIndexOutput.cs | 162 +++++++++++++
.../Intblock/FixedIntBlockIndexInput.cs | 196 ----------------
.../Intblock/FixedIntBlockIndexOutput.cs | 151 ------------
.../Intblock/VariableIntBlockIndexInput.cs | 229 -------------------
.../Intblock/VariableIntBlockIndexOutput.cs | 162 -------------
8 files changed, 738 insertions(+), 738 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs
new file mode 100644
index 0000000..991b5f0
--- /dev/null
+++ b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs
@@ -0,0 +1,196 @@
+\ufeffusing Lucene.Net.Codecs.Sep;
+using Lucene.Net.Store;
+using System.Diagnostics;
+
+namespace Lucene.Net.Codecs.IntBlock
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ // Naive int block API that writes vInts. This is
+ // expected to give poor performance; it's really only for
+ // testing the pluggability. One should typically use pfor instead.
+
+ /// <summary>
+ /// Abstract base class that reads fixed-size blocks of ints
+ /// from an IndexInput. While this is a simple approach, a
+ /// more performant approach would directly create an impl
+ /// of IntIndexInput inside Directory. Wrapping a generic
+ /// IndexInput will likely cost performance.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class FixedIntBlockIndexInput : IntIndexInput
+ {
+ private readonly IndexInput input;
+ protected readonly int m_blockSize;
+
+ public FixedIntBlockIndexInput(IndexInput @in)
+ {
+ input = @in;
+ m_blockSize = @in.ReadVInt();
+ }
+
+ public override AbstractReader GetReader()
+ {
+ var buffer = new int[m_blockSize];
+ var clone = (IndexInput)input.Clone();
+ // TODO: can this be simplified?
+ return new Reader(clone, buffer, GetBlockReader(clone, buffer));
+ }
+
+ public override void Dispose()
+ {
+ input.Dispose();
+ }
+
+ public override AbstractIndex GetIndex()
+ {
+ return new Index(this);
+ }
+
+ protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer);
+
+
+ /// <summary>
+ /// Interface for fixed-size block decoders.
+ /// <para>
+ /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>.
+ /// </para>
+ /// </summary>
+ public interface IBlockReader
+ {
+ void ReadBlock();
+ }
+
+ private class Reader : AbstractReader
+ {
+ private readonly IndexInput input;
+ private readonly IBlockReader blockReader;
+ private readonly int blockSize;
+ private readonly int[] pending;
+
+ private int upto;
+ private bool seekPending;
+ private long pendingFP;
+ private long lastBlockFP = -1;
+
+ public Reader(IndexInput input, int[] pending, IBlockReader blockReader)
+ {
+ this.input = input;
+ this.pending = pending;
+ this.blockSize = pending.Length;
+ this.blockReader = blockReader;
+ upto = blockSize;
+ }
+
+ internal virtual void Seek(long fp, int upto)
+ {
+ Debug.Assert(upto < blockSize);
+ if (seekPending || fp != lastBlockFP)
+ {
+ pendingFP = fp;
+ seekPending = true;
+ }
+ this.upto = upto;
+ }
+
+ public override int Next()
+ {
+ if (seekPending)
+ {
+ // Seek & load new block
+ input.Seek(pendingFP);
+ lastBlockFP = pendingFP;
+ blockReader.ReadBlock();
+ seekPending = false;
+ }
+ else if (upto == blockSize)
+ {
+ // Load new block
+ lastBlockFP = input.FilePointer;
+ blockReader.ReadBlock();
+ upto = 0;
+ }
+ return pending[upto++];
+ }
+ }
+
+ private class Index : AbstractIndex
+ {
+ private readonly FixedIntBlockIndexInput outerInstance;
+
+ public Index(FixedIntBlockIndexInput outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ private long fp;
+ private int upto;
+
+ public override void Read(DataInput indexIn, bool absolute)
+ {
+ if (absolute)
+ {
+ upto = indexIn.ReadVInt();
+ fp = indexIn.ReadVLong();
+ }
+ else
+ {
+ int uptoDelta = indexIn.ReadVInt();
+ if ((uptoDelta & 1) == 1)
+ {
+ // same block
+ upto += (int)((uint)uptoDelta >> 1);
+ }
+ else
+ {
+ // new block
+ upto = (int)((uint)uptoDelta >> 1);
+ fp += indexIn.ReadVLong();
+ }
+ }
+ Debug.Assert(upto < outerInstance.m_blockSize);
+ }
+
+ public override void Seek(AbstractReader other)
+ {
+ ((Reader)other).Seek(fp, upto);
+ }
+
+ public override void CopyFrom(AbstractIndex other)
+ {
+ Index idx = (Index)other;
+ fp = idx.fp;
+ upto = idx.upto;
+ }
+
+ public override AbstractIndex Clone()
+ {
+ Index other = new Index(outerInstance);
+ other.fp = fp;
+ other.upto = upto;
+ return other;
+ }
+
+ public override string ToString()
+ {
+ return "fp=" + fp + " upto=" + upto;
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs
new file mode 100644
index 0000000..544b644
--- /dev/null
+++ b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs
@@ -0,0 +1,151 @@
+\ufeffusing Lucene.Net.Codecs.Sep;
+using Lucene.Net.Store;
+using System.Diagnostics;
+
+namespace Lucene.Net.Codecs.IntBlock
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Naive int block API that writes vInts. This is
+ /// expected to give poor performance; it's really only for
+ /// testing the pluggability. One should typically use pfor instead.
+ /// </summary>
+
+
+ /// <summary>
+ /// Abstract base class that writes fixed-size blocks of ints
+ /// to an IndexOutput. While this is a simple approach, a
+ /// more performant approach would directly create an impl
+ /// of IntIndexOutput inside Directory. Wrapping a generic
+ /// IndexInput will likely cost performance.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class FixedIntBlockIndexOutput : IntIndexOutput
+ {
+ protected readonly IndexOutput m_output;
+ private readonly int blockSize;
+ protected readonly int[] m_buffer;
+ private int upto;
+
+ protected FixedIntBlockIndexOutput(IndexOutput output, int fixedBlockSize)
+ {
+ blockSize = fixedBlockSize;
+ this.m_output = output;
+ output.WriteVInt(blockSize);
+ m_buffer = new int[blockSize];
+ }
+
+ protected abstract void FlushBlock();
+
+ public override AbstractIndex GetIndex()
+ {
+ return new Index(this);
+ }
+
+ private class Index : AbstractIndex
+ {
+ private readonly FixedIntBlockIndexOutput outerInstance;
+
+ public Index(FixedIntBlockIndexOutput outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ internal long fp;
+ internal int upto;
+ internal long lastFP;
+ internal int lastUpto;
+
+ public override void Mark()
+ {
+ fp = outerInstance.m_output.FilePointer;
+ upto = outerInstance.upto;
+ }
+
+ public override void CopyFrom(AbstractIndex other, bool copyLast)
+ {
+ Index idx = (Index)other;
+ fp = idx.fp;
+ upto = idx.upto;
+ if (copyLast)
+ {
+ lastFP = fp;
+ lastUpto = upto;
+ }
+ }
+
+ public override void Write(DataOutput indexOut, bool absolute)
+ {
+ if (absolute)
+ {
+ indexOut.WriteVInt(upto);
+ indexOut.WriteVLong(fp);
+ }
+ else if (fp == lastFP)
+ {
+ // same block
+ Debug.Assert(upto >= lastUpto);
+ int uptoDelta = upto - lastUpto;
+ indexOut.WriteVInt(uptoDelta << 1 | 1);
+ }
+ else
+ {
+ // new block
+ indexOut.WriteVInt(upto << 1);
+ indexOut.WriteVLong(fp - lastFP);
+ }
+ lastUpto = upto;
+ lastFP = fp;
+ }
+
+ public override string ToString()
+ {
+ return "fp=" + fp + " upto=" + upto;
+ }
+ }
+
+ public override void Write(int v)
+ {
+ m_buffer[upto++] = v;
+ if (upto == blockSize)
+ {
+ FlushBlock();
+ upto = 0;
+ }
+ }
+
+ public override void Dispose()
+ {
+ try
+ {
+ if (upto > 0)
+ {
+ // NOTE: entries in the block after current upto are
+ // invalid
+ FlushBlock();
+ }
+ }
+ finally
+ {
+ m_output.Dispose();
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs
new file mode 100644
index 0000000..6607801
--- /dev/null
+++ b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs
@@ -0,0 +1,229 @@
+\ufeffusing Lucene.Net.Codecs.Sep;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using System.Diagnostics;
+
+namespace Lucene.Net.Codecs.IntBlock
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ // Naive int block API that writes vInts. This is
+ // expected to give poor performance; it's really only for
+ // testing the pluggability. One should typically use pfor instead.
+
+ // TODO: much of this can be shared code w/ the fixed case
+
+ /// <summary>
+ /// Abstract base class that reads variable-size blocks of ints
+ /// from an IndexInput. While this is a simple approach, a
+ /// more performant approach would directly create an impl
+ /// of IntIndexInput inside Directory. Wrapping a generic
+ /// IndexInput will likely cost performance.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class VariableIntBlockIndexInput : IntIndexInput
+ {
+ private readonly IndexInput input;
+ protected readonly int m_maxBlockSize;
+
+ protected internal VariableIntBlockIndexInput(IndexInput input)
+ {
+ this.input = input;
+ m_maxBlockSize = input.ReadInt();
+ }
+
+ public override AbstractReader GetReader()
+ {
+ var buffer = new int[m_maxBlockSize];
+ var clone = (IndexInput)input.Clone();
+ // TODO: can this be simplified?
+ return new Reader(clone, buffer, GetBlockReader(clone, buffer));
+ }
+
+ public override void Dispose()
+ {
+ input.Dispose();
+ }
+
+ public override AbstractIndex GetIndex()
+ {
+ return new Index(this);
+ }
+
+ protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer);
+
+ /// <summary>
+ /// Interface for variable-size block decoders.
+ /// <para>
+ /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>.
+ /// </para>
+ /// </summary>
+ public interface IBlockReader
+ {
+ int ReadBlock();
+ void Seek(long pos);
+ }
+
+ private class Reader : AbstractReader
+ {
+ private readonly IndexInput input;
+
+ [WritableArray]
+ public int[] Pending
+ {
+ get { return pending; }
+ }
+ private readonly int[] pending;
+ private int upto;
+
+ private bool seekPending;
+ private long pendingFP;
+ private int pendingUpto;
+ private long lastBlockFP;
+ private int blockSize;
+ private readonly IBlockReader blockReader;
+
+ public Reader(IndexInput input, int[] pending, IBlockReader blockReader)
+ {
+ this.input = input;
+ this.pending = pending;
+ this.blockReader = blockReader;
+ }
+
+ internal virtual void Seek(long fp, int upto)
+ {
+ // TODO: should we do this in real-time, not lazy?
+ pendingFP = fp;
+ pendingUpto = upto;
+ Debug.Assert(pendingUpto >= 0, "pendingUpto=" + pendingUpto);
+ seekPending = true;
+ }
+
+ private void MaybeSeek()
+ {
+ if (seekPending)
+ {
+ if (pendingFP != lastBlockFP)
+ {
+ // need new block
+ input.Seek(pendingFP);
+ blockReader.Seek(pendingFP);
+ lastBlockFP = pendingFP;
+ blockSize = blockReader.ReadBlock();
+ }
+ upto = pendingUpto;
+
+ // TODO: if we were more clever when writing the
+ // index, such that a seek point wouldn't be written
+ // until the int encoder "committed", we could avoid
+ // this (likely minor) inefficiency:
+
+ // This is necessary for int encoders that are
+ // non-causal, ie must see future int values to
+ // encode the current ones.
+ while (upto >= blockSize)
+ {
+ upto -= blockSize;
+ lastBlockFP = input.FilePointer;
+ blockSize = blockReader.ReadBlock();
+ }
+ seekPending = false;
+ }
+ }
+
+ public override int Next()
+ {
+ this.MaybeSeek();
+ if (upto == blockSize)
+ {
+ lastBlockFP = input.FilePointer;
+ blockSize = blockReader.ReadBlock();
+ upto = 0;
+ }
+
+ return pending[upto++];
+ }
+ }
+
+ private class Index : AbstractIndex
+ {
+ private readonly VariableIntBlockIndexInput outerInstance;
+
+ public Index(VariableIntBlockIndexInput outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ private long fp;
+ private int upto;
+
+ public override void Read(DataInput indexIn, bool absolute)
+ {
+ if (absolute)
+ {
+ upto = indexIn.ReadVInt();
+ fp = indexIn.ReadVLong();
+ }
+ else
+ {
+ int uptoDelta = indexIn.ReadVInt();
+ if ((uptoDelta & 1) == 1)
+ {
+ // same block
+ upto += (int)((uint)uptoDelta >> 1);
+ }
+ else
+ {
+ // new block
+ upto = (int)((uint)uptoDelta >> 1);
+ fp += indexIn.ReadVLong();
+ }
+ }
+ // TODO: we can't do this assert because non-causal
+ // int encoders can have upto over the buffer size
+ //assert upto < maxBlockSize: "upto=" + upto + " max=" + maxBlockSize;
+ }
+
+ public override string ToString()
+ {
+ return "VarIntBlock.Index fp=" + fp + " upto=" + upto + " maxBlock=" + outerInstance.m_maxBlockSize;
+ }
+
+ public override void Seek(AbstractReader other)
+ {
+ ((Reader)other).Seek(fp, upto);
+ }
+
+ public override void CopyFrom(AbstractIndex other)
+ {
+ Index idx = (Index)other;
+ fp = idx.fp;
+ upto = idx.upto;
+ }
+
+ public override AbstractIndex Clone()
+ {
+ Index other = new Index(outerInstance);
+ other.fp = fp;
+ other.upto = upto;
+ return other;
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs
new file mode 100644
index 0000000..f756777
--- /dev/null
+++ b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs
@@ -0,0 +1,162 @@
+\ufeffusing Lucene.Net.Codecs.Sep;
+using Lucene.Net.Store;
+using System.Diagnostics;
+
+namespace Lucene.Net.Codecs.IntBlock
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Naive int block API that writes vInts. This is
+ /// expected to give poor performance; it's really only for
+ /// testing the pluggability. One should typically use pfor instead.
+ /// </summary>
+
+
+ // TODO: much of this can be shared code w/ the fixed case
+
+ /// <summary>
+ /// Abstract base class that writes variable-size blocks of ints
+ /// to an IndexOutput. While this is a simple approach, a
+ /// more performant approach would directly create an impl
+ /// of IntIndexOutput inside Directory. Wrapping a generic
+ /// IndexInput will likely cost performance.
+ ///
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class VariableIntBlockIndexOutput : IntIndexOutput
+ {
+ protected readonly IndexOutput m_output;
+
+ private int upto;
+ private bool hitExcDuringWrite;
+
+ // TODO what Var-Var codecs exist in practice... and what are there blocksizes like?
+ // if its less than 128 we should set that as max and use byte?
+
+ /// <summary>
+ /// NOTE: maxBlockSize must be the maximum block size
+ /// plus the max non-causal lookahead of your codec. EG Simple9
+ /// requires lookahead=1 because on seeing the Nth value
+ /// it knows it must now encode the N-1 values before it.
+ /// </summary>
+ protected VariableIntBlockIndexOutput(IndexOutput output, int maxBlockSize)
+ {
+ this.m_output = output;
+ this.m_output.WriteInt(maxBlockSize);
+ }
+
+ /// <summary>
+ /// Called one value at a time. Return the number of
+ /// buffered input values that have been written to out.
+ /// </summary>
+ protected abstract int Add(int value);
+
+ public override AbstractIndex GetIndex()
+ {
+ return new Index(this);
+ }
+
+ private class Index : AbstractIndex
+ {
+ private readonly VariableIntBlockIndexOutput outerInstance;
+
+ public Index(VariableIntBlockIndexOutput outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ private long fp;
+ private int upto;
+ private long lastFP;
+ private int lastUpto;
+
+ public override void Mark()
+ {
+ fp = outerInstance.m_output.FilePointer;
+ upto = outerInstance.upto;
+ }
+
+ public override void CopyFrom(AbstractIndex other, bool copyLast)
+ {
+ Index idx = (Index)other;
+ fp = idx.fp;
+ upto = idx.upto;
+ if (copyLast)
+ {
+ lastFP = fp;
+ lastUpto = upto;
+ }
+ }
+
+ public override void Write(DataOutput indexOut, bool absolute)
+ {
+ Debug.Assert(upto >= 0);
+ if (absolute)
+ {
+ indexOut.WriteVInt(upto);
+ indexOut.WriteVLong(fp);
+ }
+ else if (fp == lastFP)
+ {
+ // same block
+ Debug.Assert(upto >= lastUpto);
+ int uptoDelta = upto - lastUpto;
+ indexOut.WriteVInt(uptoDelta << 1 | 1);
+ }
+ else
+ {
+ // new block
+ indexOut.WriteVInt(upto << 1);
+ indexOut.WriteVLong(fp - lastFP);
+ }
+ lastUpto = upto;
+ lastFP = fp;
+ }
+ }
+
+ public override void Write(int v)
+ {
+ hitExcDuringWrite = true;
+ upto -= Add(v) - 1;
+ hitExcDuringWrite = false;
+ Debug.Assert(upto >= 0);
+ }
+
+ public override void Dispose()
+ {
+ try
+ {
+ if (hitExcDuringWrite) return;
+
+ // stuff 0s in until the "real" data is flushed:
+ var stuffed = 0;
+ while (upto > stuffed)
+ {
+ upto -= Add(0) - 1;
+ Debug.Assert(upto >= 0);
+ stuffed += 1;
+ }
+ }
+ finally
+ {
+ m_output.Dispose();
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs
deleted file mode 100644
index 991b5f0..0000000
--- a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs
+++ /dev/null
@@ -1,196 +0,0 @@
-\ufeffusing Lucene.Net.Codecs.Sep;
-using Lucene.Net.Store;
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.IntBlock
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- // Naive int block API that writes vInts. This is
- // expected to give poor performance; it's really only for
- // testing the pluggability. One should typically use pfor instead.
-
- /// <summary>
- /// Abstract base class that reads fixed-size blocks of ints
- /// from an IndexInput. While this is a simple approach, a
- /// more performant approach would directly create an impl
- /// of IntIndexInput inside Directory. Wrapping a generic
- /// IndexInput will likely cost performance.
- ///
- /// @lucene.experimental
- /// </summary>
- public abstract class FixedIntBlockIndexInput : IntIndexInput
- {
- private readonly IndexInput input;
- protected readonly int m_blockSize;
-
- public FixedIntBlockIndexInput(IndexInput @in)
- {
- input = @in;
- m_blockSize = @in.ReadVInt();
- }
-
- public override AbstractReader GetReader()
- {
- var buffer = new int[m_blockSize];
- var clone = (IndexInput)input.Clone();
- // TODO: can this be simplified?
- return new Reader(clone, buffer, GetBlockReader(clone, buffer));
- }
-
- public override void Dispose()
- {
- input.Dispose();
- }
-
- public override AbstractIndex GetIndex()
- {
- return new Index(this);
- }
-
- protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer);
-
-
- /// <summary>
- /// Interface for fixed-size block decoders.
- /// <para>
- /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>.
- /// </para>
- /// </summary>
- public interface IBlockReader
- {
- void ReadBlock();
- }
-
- private class Reader : AbstractReader
- {
- private readonly IndexInput input;
- private readonly IBlockReader blockReader;
- private readonly int blockSize;
- private readonly int[] pending;
-
- private int upto;
- private bool seekPending;
- private long pendingFP;
- private long lastBlockFP = -1;
-
- public Reader(IndexInput input, int[] pending, IBlockReader blockReader)
- {
- this.input = input;
- this.pending = pending;
- this.blockSize = pending.Length;
- this.blockReader = blockReader;
- upto = blockSize;
- }
-
- internal virtual void Seek(long fp, int upto)
- {
- Debug.Assert(upto < blockSize);
- if (seekPending || fp != lastBlockFP)
- {
- pendingFP = fp;
- seekPending = true;
- }
- this.upto = upto;
- }
-
- public override int Next()
- {
- if (seekPending)
- {
- // Seek & load new block
- input.Seek(pendingFP);
- lastBlockFP = pendingFP;
- blockReader.ReadBlock();
- seekPending = false;
- }
- else if (upto == blockSize)
- {
- // Load new block
- lastBlockFP = input.FilePointer;
- blockReader.ReadBlock();
- upto = 0;
- }
- return pending[upto++];
- }
- }
-
- private class Index : AbstractIndex
- {
- private readonly FixedIntBlockIndexInput outerInstance;
-
- public Index(FixedIntBlockIndexInput outerInstance)
- {
- this.outerInstance = outerInstance;
- }
-
- private long fp;
- private int upto;
-
- public override void Read(DataInput indexIn, bool absolute)
- {
- if (absolute)
- {
- upto = indexIn.ReadVInt();
- fp = indexIn.ReadVLong();
- }
- else
- {
- int uptoDelta = indexIn.ReadVInt();
- if ((uptoDelta & 1) == 1)
- {
- // same block
- upto += (int)((uint)uptoDelta >> 1);
- }
- else
- {
- // new block
- upto = (int)((uint)uptoDelta >> 1);
- fp += indexIn.ReadVLong();
- }
- }
- Debug.Assert(upto < outerInstance.m_blockSize);
- }
-
- public override void Seek(AbstractReader other)
- {
- ((Reader)other).Seek(fp, upto);
- }
-
- public override void CopyFrom(AbstractIndex other)
- {
- Index idx = (Index)other;
- fp = idx.fp;
- upto = idx.upto;
- }
-
- public override AbstractIndex Clone()
- {
- Index other = new Index(outerInstance);
- other.fp = fp;
- other.upto = upto;
- return other;
- }
-
- public override string ToString()
- {
- return "fp=" + fp + " upto=" + upto;
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs
deleted file mode 100644
index 544b644..0000000
--- a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs
+++ /dev/null
@@ -1,151 +0,0 @@
-\ufeffusing Lucene.Net.Codecs.Sep;
-using Lucene.Net.Store;
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.IntBlock
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- /// <summary>
- /// Naive int block API that writes vInts. This is
- /// expected to give poor performance; it's really only for
- /// testing the pluggability. One should typically use pfor instead.
- /// </summary>
-
-
- /// <summary>
- /// Abstract base class that writes fixed-size blocks of ints
- /// to an IndexOutput. While this is a simple approach, a
- /// more performant approach would directly create an impl
- /// of IntIndexOutput inside Directory. Wrapping a generic
- /// IndexInput will likely cost performance.
- ///
- /// @lucene.experimental
- /// </summary>
- public abstract class FixedIntBlockIndexOutput : IntIndexOutput
- {
- protected readonly IndexOutput m_output;
- private readonly int blockSize;
- protected readonly int[] m_buffer;
- private int upto;
-
- protected FixedIntBlockIndexOutput(IndexOutput output, int fixedBlockSize)
- {
- blockSize = fixedBlockSize;
- this.m_output = output;
- output.WriteVInt(blockSize);
- m_buffer = new int[blockSize];
- }
-
- protected abstract void FlushBlock();
-
- public override AbstractIndex GetIndex()
- {
- return new Index(this);
- }
-
- private class Index : AbstractIndex
- {
- private readonly FixedIntBlockIndexOutput outerInstance;
-
- public Index(FixedIntBlockIndexOutput outerInstance)
- {
- this.outerInstance = outerInstance;
- }
-
- internal long fp;
- internal int upto;
- internal long lastFP;
- internal int lastUpto;
-
- public override void Mark()
- {
- fp = outerInstance.m_output.FilePointer;
- upto = outerInstance.upto;
- }
-
- public override void CopyFrom(AbstractIndex other, bool copyLast)
- {
- Index idx = (Index)other;
- fp = idx.fp;
- upto = idx.upto;
- if (copyLast)
- {
- lastFP = fp;
- lastUpto = upto;
- }
- }
-
- public override void Write(DataOutput indexOut, bool absolute)
- {
- if (absolute)
- {
- indexOut.WriteVInt(upto);
- indexOut.WriteVLong(fp);
- }
- else if (fp == lastFP)
- {
- // same block
- Debug.Assert(upto >= lastUpto);
- int uptoDelta = upto - lastUpto;
- indexOut.WriteVInt(uptoDelta << 1 | 1);
- }
- else
- {
- // new block
- indexOut.WriteVInt(upto << 1);
- indexOut.WriteVLong(fp - lastFP);
- }
- lastUpto = upto;
- lastFP = fp;
- }
-
- public override string ToString()
- {
- return "fp=" + fp + " upto=" + upto;
- }
- }
-
- public override void Write(int v)
- {
- m_buffer[upto++] = v;
- if (upto == blockSize)
- {
- FlushBlock();
- upto = 0;
- }
- }
-
- public override void Dispose()
- {
- try
- {
- if (upto > 0)
- {
- // NOTE: entries in the block after current upto are
- // invalid
- FlushBlock();
- }
- }
- finally
- {
- m_output.Dispose();
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs
deleted file mode 100644
index 6607801..0000000
--- a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs
+++ /dev/null
@@ -1,229 +0,0 @@
-\ufeffusing Lucene.Net.Codecs.Sep;
-using Lucene.Net.Store;
-using Lucene.Net.Support;
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.IntBlock
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- // Naive int block API that writes vInts. This is
- // expected to give poor performance; it's really only for
- // testing the pluggability. One should typically use pfor instead.
-
- // TODO: much of this can be shared code w/ the fixed case
-
- /// <summary>
- /// Abstract base class that reads variable-size blocks of ints
- /// from an IndexInput. While this is a simple approach, a
- /// more performant approach would directly create an impl
- /// of IntIndexInput inside Directory. Wrapping a generic
- /// IndexInput will likely cost performance.
- ///
- /// @lucene.experimental
- /// </summary>
- public abstract class VariableIntBlockIndexInput : IntIndexInput
- {
- private readonly IndexInput input;
- protected readonly int m_maxBlockSize;
-
- protected internal VariableIntBlockIndexInput(IndexInput input)
- {
- this.input = input;
- m_maxBlockSize = input.ReadInt();
- }
-
- public override AbstractReader GetReader()
- {
- var buffer = new int[m_maxBlockSize];
- var clone = (IndexInput)input.Clone();
- // TODO: can this be simplified?
- return new Reader(clone, buffer, GetBlockReader(clone, buffer));
- }
-
- public override void Dispose()
- {
- input.Dispose();
- }
-
- public override AbstractIndex GetIndex()
- {
- return new Index(this);
- }
-
- protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer);
-
- /// <summary>
- /// Interface for variable-size block decoders.
- /// <para>
- /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>.
- /// </para>
- /// </summary>
- public interface IBlockReader
- {
- int ReadBlock();
- void Seek(long pos);
- }
-
- private class Reader : AbstractReader
- {
- private readonly IndexInput input;
-
- [WritableArray]
- public int[] Pending
- {
- get { return pending; }
- }
- private readonly int[] pending;
- private int upto;
-
- private bool seekPending;
- private long pendingFP;
- private int pendingUpto;
- private long lastBlockFP;
- private int blockSize;
- private readonly IBlockReader blockReader;
-
- public Reader(IndexInput input, int[] pending, IBlockReader blockReader)
- {
- this.input = input;
- this.pending = pending;
- this.blockReader = blockReader;
- }
-
- internal virtual void Seek(long fp, int upto)
- {
- // TODO: should we do this in real-time, not lazy?
- pendingFP = fp;
- pendingUpto = upto;
- Debug.Assert(pendingUpto >= 0, "pendingUpto=" + pendingUpto);
- seekPending = true;
- }
-
- private void MaybeSeek()
- {
- if (seekPending)
- {
- if (pendingFP != lastBlockFP)
- {
- // need new block
- input.Seek(pendingFP);
- blockReader.Seek(pendingFP);
- lastBlockFP = pendingFP;
- blockSize = blockReader.ReadBlock();
- }
- upto = pendingUpto;
-
- // TODO: if we were more clever when writing the
- // index, such that a seek point wouldn't be written
- // until the int encoder "committed", we could avoid
- // this (likely minor) inefficiency:
-
- // This is necessary for int encoders that are
- // non-causal, ie must see future int values to
- // encode the current ones.
- while (upto >= blockSize)
- {
- upto -= blockSize;
- lastBlockFP = input.FilePointer;
- blockSize = blockReader.ReadBlock();
- }
- seekPending = false;
- }
- }
-
- public override int Next()
- {
- this.MaybeSeek();
- if (upto == blockSize)
- {
- lastBlockFP = input.FilePointer;
- blockSize = blockReader.ReadBlock();
- upto = 0;
- }
-
- return pending[upto++];
- }
- }
-
- private class Index : AbstractIndex
- {
- private readonly VariableIntBlockIndexInput outerInstance;
-
- public Index(VariableIntBlockIndexInput outerInstance)
- {
- this.outerInstance = outerInstance;
- }
-
- private long fp;
- private int upto;
-
- public override void Read(DataInput indexIn, bool absolute)
- {
- if (absolute)
- {
- upto = indexIn.ReadVInt();
- fp = indexIn.ReadVLong();
- }
- else
- {
- int uptoDelta = indexIn.ReadVInt();
- if ((uptoDelta & 1) == 1)
- {
- // same block
- upto += (int)((uint)uptoDelta >> 1);
- }
- else
- {
- // new block
- upto = (int)((uint)uptoDelta >> 1);
- fp += indexIn.ReadVLong();
- }
- }
- // TODO: we can't do this assert because non-causal
- // int encoders can have upto over the buffer size
- //assert upto < maxBlockSize: "upto=" + upto + " max=" + maxBlockSize;
- }
-
- public override string ToString()
- {
- return "VarIntBlock.Index fp=" + fp + " upto=" + upto + " maxBlock=" + outerInstance.m_maxBlockSize;
- }
-
- public override void Seek(AbstractReader other)
- {
- ((Reader)other).Seek(fp, upto);
- }
-
- public override void CopyFrom(AbstractIndex other)
- {
- Index idx = (Index)other;
- fp = idx.fp;
- upto = idx.upto;
- }
-
- public override AbstractIndex Clone()
- {
- Index other = new Index(outerInstance);
- other.fp = fp;
- other.upto = upto;
- return other;
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs
deleted file mode 100644
index f756777..0000000
--- a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs
+++ /dev/null
@@ -1,162 +0,0 @@
-\ufeffusing Lucene.Net.Codecs.Sep;
-using Lucene.Net.Store;
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.IntBlock
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- /// <summary>
- /// Naive int block API that writes vInts. This is
- /// expected to give poor performance; it's really only for
- /// testing the pluggability. One should typically use pfor instead.
- /// </summary>
-
-
- // TODO: much of this can be shared code w/ the fixed case
-
- /// <summary>
- /// Abstract base class that writes variable-size blocks of ints
- /// to an IndexOutput. While this is a simple approach, a
- /// more performant approach would directly create an impl
- /// of IntIndexOutput inside Directory. Wrapping a generic
- /// IndexInput will likely cost performance.
- ///
- /// @lucene.experimental
- /// </summary>
- public abstract class VariableIntBlockIndexOutput : IntIndexOutput
- {
- protected readonly IndexOutput m_output;
-
- private int upto;
- private bool hitExcDuringWrite;
-
- // TODO what Var-Var codecs exist in practice... and what are there blocksizes like?
- // if its less than 128 we should set that as max and use byte?
-
- /// <summary>
- /// NOTE: maxBlockSize must be the maximum block size
- /// plus the max non-causal lookahead of your codec. EG Simple9
- /// requires lookahead=1 because on seeing the Nth value
- /// it knows it must now encode the N-1 values before it.
- /// </summary>
- protected VariableIntBlockIndexOutput(IndexOutput output, int maxBlockSize)
- {
- this.m_output = output;
- this.m_output.WriteInt(maxBlockSize);
- }
-
- /// <summary>
- /// Called one value at a time. Return the number of
- /// buffered input values that have been written to out.
- /// </summary>
- protected abstract int Add(int value);
-
- public override AbstractIndex GetIndex()
- {
- return new Index(this);
- }
-
- private class Index : AbstractIndex
- {
- private readonly VariableIntBlockIndexOutput outerInstance;
-
- public Index(VariableIntBlockIndexOutput outerInstance)
- {
- this.outerInstance = outerInstance;
- }
-
- private long fp;
- private int upto;
- private long lastFP;
- private int lastUpto;
-
- public override void Mark()
- {
- fp = outerInstance.m_output.FilePointer;
- upto = outerInstance.upto;
- }
-
- public override void CopyFrom(AbstractIndex other, bool copyLast)
- {
- Index idx = (Index)other;
- fp = idx.fp;
- upto = idx.upto;
- if (copyLast)
- {
- lastFP = fp;
- lastUpto = upto;
- }
- }
-
- public override void Write(DataOutput indexOut, bool absolute)
- {
- Debug.Assert(upto >= 0);
- if (absolute)
- {
- indexOut.WriteVInt(upto);
- indexOut.WriteVLong(fp);
- }
- else if (fp == lastFP)
- {
- // same block
- Debug.Assert(upto >= lastUpto);
- int uptoDelta = upto - lastUpto;
- indexOut.WriteVInt(uptoDelta << 1 | 1);
- }
- else
- {
- // new block
- indexOut.WriteVInt(upto << 1);
- indexOut.WriteVLong(fp - lastFP);
- }
- lastUpto = upto;
- lastFP = fp;
- }
- }
-
- public override void Write(int v)
- {
- hitExcDuringWrite = true;
- upto -= Add(v) - 1;
- hitExcDuringWrite = false;
- Debug.Assert(upto >= 0);
- }
-
- public override void Dispose()
- {
- try
- {
- if (hitExcDuringWrite) return;
-
- // stuff 0s in until the "real" data is flushed:
- var stuffed = 0;
- while (upto > stuffed)
- {
- upto -= Add(0) - 1;
- Debug.Assert(upto >= 0);
- stuffed += 1;
- }
- }
- finally
- {
- m_output.Dispose();
- }
- }
- }
-}
\ No newline at end of file