You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2016/10/11 18:35:14 UTC
[29/47] lucenenet git commit: Fixed several bugs in Codecs.Memory
that were causing tests not to pass.
Fixed several bugs in Codecs.Memory that were causing tests not to pass.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/c574b757
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/c574b757
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/c574b757
Branch: refs/heads/master
Commit: c574b757e1d449b14e59414a410adccd6fdff959
Parents: d7dca52
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Mon Oct 10 15:12:53 2016 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Tue Oct 11 00:26:25 2016 +0700
----------------------------------------------------------------------
.../Memory/DirectDocValuesConsumer.cs | 45 +++++++++++---------
.../Memory/DirectDocValuesProducer.cs | 27 ++++++------
.../Memory/DirectPostingsFormat.cs | 4 +-
.../Memory/FSTOrdTermsReader.cs | 7 ++-
src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs | 6 ++-
.../Memory/MemoryDocValuesConsumer.cs | 17 ++++++--
.../Memory/MemoryDocValuesProducer.cs | 30 ++++++-------
7 files changed, 78 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs b/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
index 4fb2a00..45fccde 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectDocValuesConsumer.cs
@@ -48,11 +48,11 @@ namespace Lucene.Net.Codecs.Memory
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
dataExtension);
data = state.Directory.CreateOutput(dataName, state.Context);
- CodecUtil.WriteHeader(data, dataCodec, MemoryDocValuesProducer.VERSION_CURRENT);
+ CodecUtil.WriteHeader(data, dataCodec, DirectDocValuesProducer.VERSION_CURRENT);
string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
metaExtension);
meta = state.Directory.CreateOutput(metaName, state.Context);
- CodecUtil.WriteHeader(meta, metaCodec, MemoryDocValuesProducer.VERSION_CURRENT);
+ CodecUtil.WriteHeader(meta, metaCodec, DirectDocValuesProducer.VERSION_CURRENT);
success = true;
}
finally
@@ -136,7 +136,7 @@ namespace Lucene.Net.Codecs.Memory
long v;
if (nv != null)
{
- v = (long) nv;
+ v = nv.Value;
}
else
{
@@ -329,8 +329,7 @@ namespace Lucene.Net.Codecs.Memory
// (the final sum):
public virtual IEnumerator<long?> GetEnumerator()
{
- var iter = _docToOrdCount.GetEnumerator();
- return new IteratorAnonymousInnerClassHelper(this, iter);
+ return new IteratorAnonymousInnerClassHelper(this, _docToOrdCount);
}
IEnumerator IEnumerable.GetEnumerator()
@@ -340,23 +339,24 @@ namespace Lucene.Net.Codecs.Memory
private class IteratorAnonymousInnerClassHelper : IEnumerator<long?>
{
- private readonly IEnumerator<long?> _iter;
+ private readonly IEnumerator<long?> iter;
public IteratorAnonymousInnerClassHelper(IterableAnonymousInnerClassHelper outerInstance,
- IEnumerator<long?> iter)
+ IEnumerable<long?> docToOrdCount)
{
- _iter = iter;
+ this.iter = docToOrdCount.GetEnumerator();
}
private long sum;
private bool ended;
+ private long toReturn;
public long? Current
{
get
{
- return _iter.Current;
+ return toReturn;
}
}
@@ -368,17 +368,16 @@ namespace Lucene.Net.Codecs.Memory
}
}
- public virtual void Remove()
- {
- throw new NotSupportedException();
- }
+ // LUCENENET: Remove() not supported by .NET
public bool MoveNext()
{
- long toReturn = sum;
- if (_iter.MoveNext())
+ if (ended) return false;
+
+ toReturn = sum;
+ if (iter.MoveNext())
{
- long? n = _iter.Current;
+ long? n = iter.Current;
if (n.HasValue)
{
sum += n.Value;
@@ -386,18 +385,21 @@ namespace Lucene.Net.Codecs.Memory
return true;
}
- else
+ else if (!ended)
{
ended = true;
+ return true;
+ }
+ else
+ {
+ Debug.Assert(false);
return false;
}
-
- //return toReturn;
}
public void Reset()
{
- throw new NotImplementedException();
+ throw new NotSupportedException();
}
#region IDisposable Support
@@ -409,7 +411,8 @@ namespace Lucene.Net.Codecs.Memory
{
if (disposing)
{
- // TODO: dispose managed state (managed objects).
+ // TODO: dispose managed state (managed objects).
+ iter.Dispose();
}
// TODO: free unmanaged resources (unmanaged objects) and override a finalizer below.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs b/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
index 19d71d2..f98933e 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectDocValuesProducer.cs
@@ -4,6 +4,7 @@ using Lucene.Net.Index;
using Lucene.Net.Store;
using Lucene.Net.Support;
using Lucene.Net.Util;
+using System;
namespace Lucene.Net.Codecs.Memory
{
@@ -214,8 +215,8 @@ namespace Lucene.Net.Codecs.Memory
{
lock (this)
{
- var instance = numericInstances[field.Number];
- if (instance == null)
+ NumericDocValues instance;
+ if (!numericInstances.TryGetValue(field.Number, out instance))
{
// Lazy load
instance = LoadNumeric(numerics[field.Number]);
@@ -235,7 +236,8 @@ namespace Lucene.Net.Codecs.Memory
var values = new byte[entry.count];
data.ReadBytes(values, 0, entry.count);
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values));
- return new NumericDocValuesAnonymousInnerClassHelper(values);
+ // LUCENENET: IMPORTANT - some bytes are negative here, so we need to pass as sbyte
+ return new NumericDocValuesAnonymousInnerClassHelper((sbyte[])(Array)values);
}
case 2:
@@ -278,9 +280,9 @@ namespace Lucene.Net.Codecs.Memory
private class NumericDocValuesAnonymousInnerClassHelper : NumericDocValues
{
- private readonly byte[] values;
+ private readonly sbyte[] values;
- public NumericDocValuesAnonymousInnerClassHelper(byte[] values)
+ public NumericDocValuesAnonymousInnerClassHelper(sbyte[] values)
{
this.values = values;
}
@@ -340,8 +342,8 @@ namespace Lucene.Net.Codecs.Memory
{
lock (this)
{
- var instance = binaryInstances[field.Number];
- if (instance == null)
+ BinaryDocValues instance;
+ if (!binaryInstances.TryGetValue(field.Number, out instance))
{
// Lazy load
instance = LoadBinary(binaries[field.Number]);
@@ -393,8 +395,8 @@ namespace Lucene.Net.Codecs.Memory
{
lock (this)
{
- var instance = sortedInstances[field.Number];
- if (instance == null)
+ SortedDocValues instance;
+ if (!sortedInstances.TryGetValue(field.Number, out instance))
{
// Lazy load
instance = LoadSorted(field);
@@ -457,9 +459,9 @@ namespace Lucene.Net.Codecs.Memory
{
lock (this)
{
- var instance = sortedSetInstances[field.Number];
var entry = sortedSets[field.Number];
- if (instance == null)
+ SortedSetRawValues instance;
+ if (!sortedSetInstances.TryGetValue(field.Number, out instance))
{
// Lazy load
instance = LoadSortedSet(entry);
@@ -563,8 +565,7 @@ namespace Lucene.Net.Codecs.Memory
Bits instance;
lock (this)
{
- instance = docsWithFieldInstances[fieldNumber];
- if (instance == null)
+ if (!docsWithFieldInstances.TryGetValue(fieldNumber, out instance))
{
var data = (IndexInput)this.data.Clone();
data.Seek(offset);
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs b/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
index 378399d..de8a586 100644
--- a/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
+++ b/src/Lucene.Net.Codecs/Memory/DirectPostingsFormat.cs
@@ -334,7 +334,7 @@ namespace Lucene.Net.Codecs.Memory
terms = new TermAndSkip[numTerms];
termOffsets = new int[1 + numTerms];
- sbyte[] termBytes = new sbyte[1024];
+ byte[] termBytes = new byte[1024];
this.minSkipCount = minSkipCount;
@@ -622,7 +622,7 @@ namespace Lucene.Net.Codecs.Memory
return termLen - other.Length;
}
- private void SetSkips(int termOrd, sbyte[] termBytes)
+ private void SetSkips(int termOrd, byte[] termBytes)
{
int termLength = termOffsets[termOrd + 1] - termOffsets[termOrd];
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
index 86afc3e..2850c3a 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTOrdTermsReader.cs
@@ -84,8 +84,11 @@ namespace Lucene.Net.Codecs.Memory
var index = new FST<long?>(indexIn, PositiveIntOutputs.Singleton);
var current = new TermsReader(this, fieldInfo, blockIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
-
- var previous = fields[fieldInfo.Name] = current;
+ TermsReader previous;
+ // LUCENENET NOTE: This simulates a put operation in Java,
+ // getting the prior value first before setting it.
+ fields.TryGetValue(fieldInfo.Name, out previous);
+ fields[fieldInfo.Name] = current;
CheckFieldSummary(state.SegmentInfo, indexIn, blockIn, current, previous);
}
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
index 0077054..93d63c7 100644
--- a/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
+++ b/src/Lucene.Net.Codecs/Memory/FSTTermsReader.cs
@@ -93,7 +93,11 @@ namespace Lucene.Net.Codecs.Memory
int docCount = @in.ReadVInt();
int longsSize = @in.ReadVInt();
TermsReader current = new TermsReader(this, fieldInfo, @in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize);
- TermsReader previous = fields[fieldInfo.Name] = current;
+ TermsReader previous;
+ // LUCENENET NOTE: This simulates a put operation in Java,
+ // getting the prior value first before setting it.
+ fields.TryGetValue(fieldInfo.Name, out previous);
+ fields[fieldInfo.Name] = current;
CheckFieldSummary(state.SegmentInfo, @in, current, previous);
}
success = true;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
index 18936e8..1c6a748 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesConsumer.cs
@@ -102,7 +102,16 @@ namespace Lucene.Net.Codecs.Memory
long count = 0;
foreach (var nv in values)
{
- long v = nv.Value;
+ long v;
+ if (nv == null)
+ {
+ v = 0;
+ missing = true;
+ }
+ else
+ {
+ v = nv.Value;
+ }
if (gcd != 1)
{
@@ -218,7 +227,7 @@ namespace Lucene.Net.Codecs.Memory
var writer = new BlockPackedWriter(data, MemoryDocValuesProducer.BLOCK_SIZE);
foreach (var nv in values)
{
- writer.Add(nv.Value);
+ writer.Add(nv.GetValueOrDefault());
}
writer.Finish();
}
@@ -494,12 +503,12 @@ namespace Lucene.Net.Codecs.Memory
public void Dispose()
{
- throw new NotImplementedException();
+ // nothing to do
}
public void Reset()
{
- throw new NotImplementedException();
+ throw new NotSupportedException();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c574b757/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
index ec82d39..2703f5e 100644
--- a/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
+++ b/src/Lucene.Net.Codecs/Memory/MemoryDocValuesProducer.cs
@@ -208,8 +208,8 @@ namespace Lucene.Net.Codecs.Memory
{
lock (this)
{
- NumericDocValues instance = numericInstances[field.Number];
- if (instance == null)
+ NumericDocValues instance;
+ if (!numericInstances.TryGetValue(field.Number, out instance))
{
instance = LoadNumeric(field);
numericInstances[field.Number] = instance;
@@ -265,7 +265,8 @@ namespace Lucene.Net.Codecs.Memory
var bytes = new byte[maxDoc];
data.ReadBytes(bytes, 0, bytes.Length);
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(bytes));
- return new NumericDocValuesAnonymousInnerClassHelper2(this, bytes);
+ // LUCENENET: IMPORTANT - some bytes are negative here, so we need to pass as sbyte
+ return new NumericDocValuesAnonymousInnerClassHelper2(this, (sbyte[])(Array)bytes);
case GCD_COMPRESSED:
long min = data.ReadLong();
long mult = data.ReadLong();
@@ -303,9 +304,9 @@ namespace Lucene.Net.Codecs.Memory
private class NumericDocValuesAnonymousInnerClassHelper2 : NumericDocValues
{
private readonly MemoryDocValuesProducer outerInstance;
- private readonly byte[] bytes;
+ private readonly sbyte[] bytes;
- public NumericDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, byte[] bytes)
+ public NumericDocValuesAnonymousInnerClassHelper2(MemoryDocValuesProducer outerInstance, sbyte[] bytes)
{
this.outerInstance = outerInstance;
this.bytes = bytes;
@@ -341,8 +342,8 @@ namespace Lucene.Net.Codecs.Memory
{
lock (this)
{
- BinaryDocValues instance = binaryInstances[field.Number];
- if (instance == null)
+ BinaryDocValues instance;
+ if (!binaryInstances.TryGetValue(field.Number, out instance))
{
instance = LoadBinary(field);
binaryInstances[field.Number] = instance;
@@ -425,8 +426,7 @@ namespace Lucene.Net.Codecs.Memory
FST<long?> instance;
lock (this)
{
- instance = fstInstances[field.Number];
- if (instance == null)
+ if (!fstInstances.TryGetValue(field.Number, out instance))
{
data.Seek(entry.offset);
instance = new FST<long?>(data, PositiveIntOutputs.Singleton);
@@ -541,8 +541,7 @@ namespace Lucene.Net.Codecs.Memory
FST<long?> instance;
lock (this)
{
- instance = fstInstances[field.Number];
- if (instance == null)
+ if (!fstInstances.TryGetValue(field.Number, out instance))
{
data.Seek(entry.offset);
instance = new FST<long?>(data, PositiveIntOutputs.Singleton);
@@ -683,8 +682,7 @@ namespace Lucene.Net.Codecs.Memory
Bits instance;
lock (this)
{
- instance = docsWithFieldInstances[fieldNumber];
- if (instance == null)
+ if (!docsWithFieldInstances.TryGetValue(fieldNumber, out instance))
{
var data = (IndexInput)this.data.Clone();
data.Seek(offset);
@@ -723,8 +721,10 @@ namespace Lucene.Net.Codecs.Memory
protected override void Dispose(bool disposing)
{
- data.Dispose();
- base.Dispose();
+ if (disposing)
+ {
+ data.Dispose();
+ }
}
internal class NumericEntry