You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/05/16 14:28:23 UTC
[03/10] lucenenet git commit: Added test for compatibility with
Lucene 4.8.0 index that has binary doc values.
Added test for compatibility with Lucene 4.8.0 index that has binary doc values.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/afb23de6
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/afb23de6
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/afb23de6
Branch: refs/heads/master
Commit: afb23de6b6c46f0a9f0a58c86aa07a1c7eb3b6f2
Parents: 0cc3cfe
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Tue May 16 17:02:35 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Tue May 16 19:17:11 2017 +0700
----------------------------------------------------------------------
.../Index/TestBinaryDocValuesUpdates.cs | 89 +++++++++++++++++++
.../Index/index.48.w-binary-doc-values.zip | Bin 0 -> 39790 bytes
.../Index/taxo.48.w-binary-doc-values.zip | Bin 0 -> 4869 bytes
src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 2 +
src/Lucene.Net.Tests/project.json | 2 +
5 files changed, 93 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs b/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
index 7f6a817..3b5e738 100644
--- a/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
+++ b/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
@@ -1,13 +1,19 @@
using Lucene.Net.Attributes;
using Lucene.Net.Codecs;
using Lucene.Net.Documents;
+using Lucene.Net.Facet;
+using Lucene.Net.Facet.Taxonomy;
+using Lucene.Net.Facet.Taxonomy.Directory;
using Lucene.Net.Randomized.Generators;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
using Lucene.Net.Support;
using Lucene.Net.Support.Threading;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.IO;
+using System.Reflection;
using System.Threading;
namespace Lucene.Net.Index
@@ -1776,5 +1782,88 @@ namespace Lucene.Net.Index
dir.Dispose();
}
+
+ /// <summary>
+ /// Using the exact translation from Lucene, we had an issue where loss of float precision was causing
+ /// index corruption when using binary doc values in combination with a 32 bit application to write the index.
+ /// Consequently, a 64 bit application could not read an index generated by a 32 bit application and a
+ /// 32 bit application could not read an index written by Java Lucene (regardless of bitness).
+ /// <para/>
+ /// This test is to verify that the current test environment (be it 32 or 64 bit) can read an index with
+ /// binary doc values that was written by Lucene in Java.
+ /// <para/>
+ /// To be thorough, we should setup the testing to run both in 32 bit and 64 bit on each platform. It would
+ /// be better if we could somehow generate the file under one bitness and consume it under the other, but
+ /// that would be difficult to set up. So, we are just reading a known good index and if we cannot read it
+ /// we know there is a problem. If the reading is fixed and the writing is different, it will fail several tests anyway.
+ /// </summary>
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestReadIndexBitness()
+ {
+ string[] alphabet = new string[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };
+ int iterations = 500;
+ FacetsConfig facetConfig = GetFacetConfig();
+
+ // Unzip index
+ DirectoryInfo indexDir = CreateTempDir("index");
+ using (Stream zipFileStream = this.GetType().GetTypeInfo().Assembly.FindAndGetManifestResourceStream(GetType(), "index.48.w-binary-doc-values.zip"))
+ {
+ TestUtil.Unzip(zipFileStream, indexDir);
+ }
+
+ // Unzip taxonomy
+ DirectoryInfo taxoDir = CreateTempDir("taxo");
+ using (Stream zipFileStream = this.GetType().GetTypeInfo().Assembly.FindAndGetManifestResourceStream(GetType(), "taxo.48.w-binary-doc-values.zip"))
+ {
+ TestUtil.Unzip(zipFileStream, taxoDir);
+ }
+
+
+ SimpleFSDirectory nFsd = new SimpleFSDirectory(indexDir);
+ IndexReader indexReader = DirectoryReader.Open(nFsd);
+ IndexSearcher searcher = new IndexSearcher(indexReader);
+
+ DirectoryTaxonomyReader taxoReader =
+ new DirectoryTaxonomyReader(FSDirectory.Open(taxoDir));
+
+ try
+ {
+ foreach (string letter in alphabet)
+ {
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new TermQuery(new Term("Field1", letter)), c);
+
+ Facets facets = new FastTaxonomyFacetCounts(taxoReader, facetConfig, c);
+ FacetResult result = facets.GetTopChildren(int.MaxValue, "facetField1");
+
+ assertEquals(iterations, System.Convert.ToInt32(result.LabelValues[0].Value));
+
+ FacetResult result2 = facets.GetTopChildren(int.MaxValue, "facetField2");
+
+
+ for (int i = 0; i < iterations; i++)
+ {
+ assertEquals(i, System.Convert.ToInt32(result2.LabelValues[i].Label));
+ }
+ }
+
+ System.Console.WriteLine("Success");
+ }
+ finally
+ {
+ IOUtils.Close(indexReader, taxoReader, nFsd);
+ }
+ }
+
+ private FacetsConfig GetFacetConfig()
+ {
+ FacetsConfig facetConfig = new FacetsConfig();
+ FacetsConfig config = new FacetsConfig();
+
+ config.SetRequireDimCount("facetField1", true);
+ config.SetRequireDimCount("facetField2", true);
+ return facetConfig;
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip b/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip
new file mode 100644
index 0000000..c18e39e
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip b/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip
new file mode 100644
index 0000000..413cb54
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
index bc1008b..a913b37 100644
--- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
+++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
@@ -113,6 +113,8 @@
<EmbeddedResource Include="Index\unsupported.24.nocfs.zip" />
<EmbeddedResource Include="Index\unsupported.29.cfs.zip" />
<EmbeddedResource Include="Index\unsupported.29.nocfs.zip" />
+ <EmbeddedResource Include="Index\index.48.w-binary-doc-values.zip" />
+ <EmbeddedResource Include="Index\taxo.48.w-binary-doc-values.zip" />
<None Include="Lucene.Net.snk" />
<None Include="Lucene.Net.Tests.project.json" />
</ItemGroup>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/project.json b/src/Lucene.Net.Tests/project.json
index c8b25ae..b1cf1a4 100644
--- a/src/Lucene.Net.Tests/project.json
+++ b/src/Lucene.Net.Tests/project.json
@@ -33,7 +33,9 @@
"Index/index.45.nocfs.zip",
"Index/index.461.cfs.zip",
"Index/index.461.nocfs.zip",
+ "Index/index.48.w-binary-doc-values.zip",
"Index/moreterms.40.zip",
+ "Index/taxo.48.w-binary-doc-values.zip",
"Index/unsupported.19.cfs.zip",
"Index/unsupported.19.nocfs.zip",
"Index/unsupported.20.cfs.zip",