You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/08/06 17:58:59 UTC
[01/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Repository: lucenenet
Updated Branches:
refs/heads/master ea879c611 -> 4782065ca
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/test-mapping-ISOLatin1Accent-partial.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/test-mapping-ISOLatin1Accent-partial.txt b/src/Lucene.Net.Tests.Benchmark/ByTask/test-mapping-ISOLatin1Accent-partial.txt
new file mode 100644
index 0000000..0ff17db
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/test-mapping-ISOLatin1Accent-partial.txt
@@ -0,0 +1,30 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Syntax:
+# "source" => "target"
+# "source".length() > 0 (source cannot be empty.)
+# "target".length() >= 0 (target can be empty.)
+
+# example:
+# "À" => "A"
+# "\u00C0" => "A"
+# "\u00C0" => "\u0041"
+# "ß" => "ss"
+# "\t" => " "
+# "\n" => ""
+
+# è => e
+"\u00E8" => "e"
+
+# é => e
+"\u00E9" => "e"
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Conf/ConfLoader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Conf/ConfLoader.cs b/src/Lucene.Net.Tests.Benchmark/Conf/ConfLoader.cs
new file mode 100644
index 0000000..51c3010
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Conf/ConfLoader.cs
@@ -0,0 +1,28 @@
+// LUCENENET specific - not used. This was to specify what directory the files
+// were in in Lucene. But we went the route of zipping the files and adding them
+// as an embedded resource, and then unpacking them to a known location for the test.
+
+//namespace Lucene.Net.Benchmarks.Conf
+//{
+// /*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+
+// public class ConfLoader
+// {
+// // don't mind me, I load .alg files
+// }
+//}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
new file mode 100644
index 0000000..c57a59f
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+ <ProjectGuid>{9257F543-44E2-4DB6-8B27-A8A354C13E5B}</ProjectGuid>
+ <OutputType>Library</OutputType>
+ <AppDesignerFolder>Properties</AppDesignerFolder>
+ <RootNamespace>Lucene.Net.Benchmarks</RootNamespace>
+ <AssemblyName>Lucene.Net.Tests.Benchmark</AssemblyName>
+ <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
+ <FileAlignment>512</FileAlignment>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+ <DebugSymbols>true</DebugSymbols>
+ <DebugType>full</DebugType>
+ <Optimize>false</Optimize>
+ <OutputPath>bin\Debug\</OutputPath>
+ <DefineConstants>DEBUG;TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
+ <OutputPath>bin\Release\</OutputPath>
+ <DefineConstants>TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <PropertyGroup>
+ <DefineConstants>$(DefineConstants);FEATURE_SERIALIZABLE</DefineConstants>
+ </PropertyGroup>
+ <ItemGroup>
+ <Reference Include="System" />
+ <Reference Include="System.Core" />
+ <Reference Include="System.Xml.Linq" />
+ <Reference Include="System.Data.DataSetExtensions" />
+ <Reference Include="Microsoft.CSharp" />
+ <Reference Include="System.Data" />
+ <Reference Include="System.Net.Http" />
+ <Reference Include="System.Xml" />
+ </ItemGroup>
+ <ItemGroup>
+ <Compile Include="BenchmarkTestCase.cs" />
+ <Compile Include="ByTask\Feeds\DocMakerTest.cs" />
+ <Compile Include="ByTask\Feeds\EnwikiContentSourceTest.cs" />
+ <Compile Include="ByTask\Feeds\LineDocSourceTest.cs" />
+ <Compile Include="ByTask\Feeds\TestHtmlParser.cs" />
+ <Compile Include="ByTask\Feeds\TrecContentSourceTest.cs" />
+ <Compile Include="ByTask\Tasks\AddIndexesTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\Alt\AltPackageTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\Alt\AltTestTask.cs" />
+ <Compile Include="ByTask\Tasks\CommitIndexTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\CountingHighlighterTestTask.cs" />
+ <Compile Include="ByTask\Tasks\CountingSearchTestTask.cs" />
+ <Compile Include="ByTask\Tasks\CreateIndexTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\PerfTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\SearchWithSortTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\WriteEnwikiLineDocTaskTest.cs" />
+ <Compile Include="ByTask\Tasks\WriteLineDocTaskTest.cs" />
+ <Compile Include="ByTask\TestPerfTasksLogic.cs" />
+ <Compile Include="ByTask\TestPerfTasksParse.cs" />
+ <Compile Include="ByTask\Utils\StreamUtilsTest.cs" />
+ <Compile Include="ByTask\Utils\TestConfig.cs" />
+ <Compile Include="Conf\ConfLoader.cs" />
+ <Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="Quality\TestQualityRun.cs" />
+ <Compile Include="Support\TestApiConsistency.cs" />
+ <Compile Include="Support\TestExceptionSerialization.cs" />
+ </ItemGroup>
+ <ItemGroup>
+ <EmbeddedResource Include="ByTask\Feeds\trecdocs.zip" />
+ <EmbeddedResource Include="ByTask\conf.zip" />
+ <None Include="Lucene.Net.Tests.Benchmark.project.json" />
+ <EmbeddedResource Include="Quality\reuters.578.lines.txt.bz2" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\Lucene.Net.Analysis.Common\Lucene.Net.Analysis.Common.csproj">
+ <Project>{4ADD0BBC-B900-4715-9526-D871DE8EEA64}</Project>
+ <Name>Lucene.Net.Analysis.Common</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Benchmark\Lucene.Net.Benchmark.csproj">
+ <Project>{edc77cb4-597f-4818-8c83-3c006d12c384}</Project>
+ <Name>Lucene.Net.Benchmark</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Facet\Lucene.Net.Facet.csproj">
+ <Project>{48F7884A-9454-4E88-8413-9D35992CB440}</Project>
+ <Name>Lucene.Net.Facet</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Highlighter\Lucene.Net.Highlighter.csproj">
+ <Project>{E9E769EA-8504-44BC-8DC9-CCF958765F8F}</Project>
+ <Name>Lucene.Net.Highlighter</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.ICU\Lucene.Net.ICU.csproj">
+ <Project>{349CB7C9-7534-4E1D-9B0A-5521441AF0AE}</Project>
+ <Name>Lucene.Net.ICU</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.TestFramework\Lucene.Net.TestFramework.csproj">
+ <Project>{B2C0D749-CE34-4F62-A15E-00CB2FF5DDB3}</Project>
+ <Name>Lucene.Net.TestFramework</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net\Lucene.Net.csproj">
+ <Project>{5d4ad9be-1ffb-41ab-9943-25737971bf57}</Project>
+ <Name>Lucene.Net</Name>
+ </ProjectReference>
+ </ItemGroup>
+ <ItemGroup>
+ <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+ </ItemGroup>
+ <ItemGroup>
+ <EmbeddedResource Include="Quality\trecQRels.txt" />
+ <EmbeddedResource Include="Quality\trecTopics.txt" />
+ </ItemGroup>
+ <ItemGroup>
+ <EmbeddedResource Include="ByTask\reuters.first20.lines.txt" />
+ <EmbeddedResource Include="ByTask\test-mapping-ISOLatin1Accent-partial.txt" />
+ </ItemGroup>
+ <ItemGroup />
+ <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+ <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Microsoft.Common.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
new file mode 100644
index 0000000..41388a7
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
@@ -0,0 +1,13 @@
+{
+ "runtimes": {
+ "win": {}
+ },
+ "dependencies": {
+ "icu.net": "54.1.1-alpha",
+ "NUnit": "3.5.0",
+ "SharpZipLib": "0.86.0"
+ },
+ "frameworks": {
+ "net451": {}
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs b/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..2e12258
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Lucene.Net.Tests.Benchmark")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Lucene.Net.Tests.Benchmark")]
+[assembly: AssemblyCopyright("Copyright © 2017")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("9257f543-44e2-4db6-8b27-a8a354c13e5b")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Quality/TestQualityRun.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Quality/TestQualityRun.cs b/src/Lucene.Net.Tests.Benchmark/Quality/TestQualityRun.cs
new file mode 100644
index 0000000..752f28b
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Quality/TestQualityRun.cs
@@ -0,0 +1,210 @@
+using Lucene.Net.Benchmarks.Quality.Trec;
+using Lucene.Net.Benchmarks.Quality.Utils;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using NUnit.Framework;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.Quality
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test that quality run does its job.
+ /// <para/>
+ /// NOTE: if the default scoring or StandardAnalyzer is changed, then
+ /// this test will not work correctly, as it does not dynamically
+ /// generate its test trec topics/qrels!
+ /// </summary>
+ public class TestQualityRun : BenchmarkTestCase
+ {
+ public override void SetUp()
+ {
+ base.SetUp();
+ copyToWorkDir("reuters.578.lines.txt.bz2");
+ }
+
+ [Test]
+ public void TestTrecQuality()
+ {
+ // first create the partial reuters index
+ createReutersIndex();
+
+
+ int maxResults = 1000;
+ String docNameField = "doctitle"; // orig docID is in the linedoc format title
+
+ TextWriter logger = VERBOSE ? Console.Out : null;
+
+ // prepare topics
+ Stream topics = GetType().getResourceAsStream("trecTopics.txt");
+ TrecTopicsReader qReader = new TrecTopicsReader();
+ QualityQuery[] qqs = qReader.ReadQueries(new StreamReader(topics, Encoding.UTF8));
+
+ // prepare judge
+ Stream qrels = GetType().getResourceAsStream("trecQRels.txt");
+ IJudge judge = new TrecJudge(new StreamReader(qrels, Encoding.UTF8));
+
+ // validate topics & judgments match each other
+ judge.ValidateData(qqs, logger);
+
+ Store.Directory dir = NewFSDirectory(new DirectoryInfo(System.IO.Path.Combine(getWorkDir().FullName, "index")));
+ IndexReader reader = DirectoryReader.Open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ IQualityQueryParser qqParser = new SimpleQQParser("title", "body");
+ QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
+
+ SubmissionReport submitLog = VERBOSE ? new SubmissionReport(logger, "TestRun") : null;
+ qrun.MaxResults = (maxResults);
+ QualityStats[] stats = qrun.Execute(judge, submitLog, logger);
+
+ // --------- verify by the way judgments were altered for this test:
+ // for some queries, depending on m = qnum % 8
+ // m==0: avg_precision and recall are hurt, by marking fake docs as relevant
+ // m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
+ // m==2: all precision, precision_at_n and recall are hurt.
+ // m>=3: these queries remain perfect
+ for (int i = 0; i < stats.Length; i++)
+ {
+ QualityStats s = stats[i];
+ switch (i % 8)
+ {
+
+ case 0:
+ assertTrue("avg-p should be hurt: " + s.GetAvp(), 1.0 > s.GetAvp());
+ assertTrue("recall should be hurt: " + s.Recall, 1.0 > s.Recall);
+ for (int j = 1; j <= QualityStats.MAX_POINTS; j++)
+ {
+ assertEquals("p_at_" + j + " should be perfect: " + s.GetPrecisionAt(j), 1.0, s.GetPrecisionAt(j), 1E-2);
+ }
+ break;
+
+ case 1:
+ assertTrue("avg-p should be hurt", 1.0 > s.GetAvp());
+ assertEquals("recall should be perfect: " + s.Recall, 1.0, s.Recall, 1E-2);
+ for (int j = 1; j <= QualityStats.MAX_POINTS; j++)
+ {
+ assertTrue("p_at_" + j + " should be hurt: " + s.GetPrecisionAt(j), 1.0 > s.GetPrecisionAt(j));
+ }
+ break;
+
+ case 2:
+ assertTrue("avg-p should be hurt: " + s.GetAvp(), 1.0 > s.GetAvp());
+ assertTrue("recall should be hurt: " + s.Recall, 1.0 > s.Recall);
+ for (int j = 1; j <= QualityStats.MAX_POINTS; j++)
+ {
+ assertTrue("p_at_" + j + " should be hurt: " + s.GetPrecisionAt(j), 1.0 > s.GetPrecisionAt(j));
+ }
+ break;
+
+ default:
+ {
+ assertEquals("avg-p should be perfect: " + s.GetAvp(), 1.0, s.GetAvp(), 1E-2);
+ assertEquals("recall should be perfect: " + s.Recall, 1.0, s.Recall, 1E-2);
+ for (int j = 1; j <= QualityStats.MAX_POINTS; j++)
+ {
+ assertEquals("p_at_" + j + " should be perfect: " + s.GetPrecisionAt(j), 1.0, s.GetPrecisionAt(j), 1E-2);
+ }
+ break;
+ }
+
+ }
+ }
+
+ QualityStats avg = QualityStats.Average(stats);
+ if (logger != null)
+ {
+ avg.Log("Average statistis:", 1, logger, " ");
+ }
+
+
+ assertTrue("mean avg-p should be hurt: " + avg.GetAvp(), 1.0 > avg.GetAvp());
+ assertTrue("avg recall should be hurt: " + avg.Recall, 1.0 > avg.Recall);
+ for (int j = 1; j <= QualityStats.MAX_POINTS; j++)
+ {
+ assertTrue("avg p_at_" + j + " should be hurt: " + avg.GetPrecisionAt(j), 1.0 > avg.GetPrecisionAt(j));
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public void TestTrecTopicsReader()
+ {
+ // prepare topics
+ Stream topicsFile = GetType().getResourceAsStream("trecTopics.txt");
+ TrecTopicsReader qReader = new TrecTopicsReader();
+ QualityQuery[] qqs = qReader.ReadQueries(
+ new StreamReader(topicsFile, Encoding.UTF8));
+
+
+ assertEquals(20, qqs.Length);
+
+ QualityQuery qq = qqs[0];
+ assertEquals("statement months total 1987", qq.GetValue("title"));
+ assertEquals("Topic 0 Description Line 1 Topic 0 Description Line 2",
+ qq.GetValue("description"));
+ assertEquals("Topic 0 Narrative Line 1 Topic 0 Narrative Line 2",
+ qq.GetValue("narrative"));
+
+ qq = qqs[1];
+ assertEquals("agreed 15 against five", qq.GetValue("title"));
+ assertEquals("Topic 1 Description Line 1 Topic 1 Description Line 2",
+ qq.GetValue("description"));
+ assertEquals("Topic 1 Narrative Line 1 Topic 1 Narrative Line 2",
+ qq.GetValue("narrative"));
+
+ qq = qqs[19];
+ assertEquals("20 while common week", qq.GetValue("title"));
+ assertEquals("Topic 19 Description Line 1 Topic 19 Description Line 2",
+ qq.GetValue("description"));
+ assertEquals("Topic 19 Narrative Line 1 Topic 19 Narrative Line 2",
+ qq.GetValue("narrative"));
+ }
+
+ // use benchmark logic to create the mini Reuters index
+ private void createReutersIndex()
+ {
+ // 1. alg definition
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "analyzer=Lucene.Net.Analysis.Standard.ClassicAnalyzer, Lucene.Net.Analysis.Common",
+ "docs.file=" + getWorkDirResourcePath("reuters.578.lines.txt.bz2"),
+ "content.source.log.step=2500",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=FSDirectory",
+ "doc.stored=true",
+ "doc.tokenized=true",
+ "# ----- alg ",
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : *",
+ "CloseIndex",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ execBenchmark(algLines);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Quality/reuters.578.lines.txt.bz2
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Quality/reuters.578.lines.txt.bz2 b/src/Lucene.Net.Tests.Benchmark/Quality/reuters.578.lines.txt.bz2
new file mode 100644
index 0000000..1fd8d54
Binary files /dev/null and b/src/Lucene.Net.Tests.Benchmark/Quality/reuters.578.lines.txt.bz2 differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Quality/trecQRels.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Quality/trecQRels.txt b/src/Lucene.Net.Tests.Benchmark/Quality/trecQRels.txt
new file mode 100644
index 0000000..13c2d77
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Quality/trecQRels.txt
@@ -0,0 +1,723 @@
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+# ------------------------------------------------------------
+# Format:
+#
+# qnum 0 doc-name is-relevant
+#
+#
+# The origin of this file was created using
+# utils.QualityQueriesFinder, so all queries
+# would have perfect 1.0 for all meassures.
+#
+# To make it suitable for testing it was modified
+# for some queries, depending on m = qnum % 8
+# m==0: avg_precision and recall are hurt, by marking fake docs as relevant
+# m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
+# m==2: all precision, precision_at_n and recall are hurt.
+# m>=3: these queries remain perfect
+# ------------------------------------------------------------
+
+# --- m==0: avg_precision and recall are hurt, by marking fake docs as relevant
+
+0 0 fakedoc1 1
+0 0 fakedoc2 1
+0 0 fakedoc3 1
+0 0 fakedoc4 1
+
+0 0 doc18211 1
+0 0 doc20192 1
+0 0 doc7401 1
+0 0 doc11285 1
+0 0 doc20647 1
+0 0 doc3057 1
+0 0 doc12431 1
+0 0 doc4989 1
+0 0 doc17324 1
+0 0 doc4030 1
+0 0 doc4290 1
+0 0 doc3462 1
+0 0 doc15313 1
+0 0 doc10303 1
+0 0 doc1893 1
+0 0 doc5008 1
+0 0 doc14634 1
+0 0 doc5471 1
+0 0 doc17904 1
+0 0 doc7168 1
+0 0 doc21275 1
+0 0 doc9011 1
+0 0 doc17546 1
+0 0 doc9102 1
+0 0 doc13199 1
+
+# --- m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
+
+1 0 doc9857 0
+1 0 doc16846 1
+1 0 doc4320 1
+1 0 doc9501 0
+1 0 doc10159 1
+1 0 doc16642 1
+1 0 doc17536 0
+1 0 doc17571 1
+1 0 doc18728 1
+1 0 doc18828 1
+1 0 doc19108 0
+1 0 doc9940 1
+1 0 doc11852 1
+1 0 doc7430 0
+1 0 doc19162 1
+1 0 doc1743 1
+1 0 doc2137 1
+1 0 doc7611 1
+1 0 doc8072 1
+1 0 doc12764 1
+1 0 doc2593 1
+1 0 doc11088 1
+1 0 doc931 1
+1 0 doc7673 1
+1 0 doc12941 1
+1 0 doc11797 1
+1 0 doc11831 1
+1 0 doc13162 1
+1 0 doc4423 1
+1 0 doc5217 1
+
+# ---- m==2: all precision, precision_at_n and recall are hurt.
+
+2 0 fakedoc1 1
+2 0 fakedoc2 1
+2 0 fakedoc3 1
+2 0 fakedoc4 1
+
+2 0 doc3137 0
+2 0 doc7142 0
+2 0 doc13667 0
+2 0 doc13171 0
+2 0 doc13372 1
+2 0 doc21415 1
+2 0 doc16298 1
+2 0 doc14957 1
+2 0 doc153 1
+2 0 doc16092 1
+2 0 doc16096 1
+2 0 doc21303 1
+2 0 doc18681 1
+2 0 doc20756 1
+2 0 doc355 1
+2 0 doc13395 1
+2 0 doc5009 1
+2 0 doc17164 1
+2 0 doc13162 1
+2 0 doc11757 1
+2 0 doc9637 1
+2 0 doc18087 1
+2 0 doc4593 1
+2 0 doc4677 1
+2 0 doc20865 1
+2 0 doc8556 1
+2 0 doc2578 1
+2 0 doc1163 1
+2 0 doc3797 1
+2 0 doc11094 1
+
+
+3 0 doc19578 1
+3 0 doc14860 1
+3 0 doc7235 1
+3 0 doc20590 1
+3 0 doc17933 1
+3 0 doc9384 1
+3 0 doc10783 1
+3 0 doc1963 1
+3 0 doc18356 1
+3 0 doc13254 1
+3 0 doc18402 1
+3 0 doc15241 1
+3 0 doc3303 1
+3 0 doc8868 1
+3 0 doc18520 1
+3 0 doc4650 1
+3 0 doc4727 1
+3 0 doc21518 1
+3 0 doc5060 1
+3 0 doc7587 1
+3 0 doc2990 1
+3 0 doc8042 1
+3 0 doc6304 1
+3 0 doc13223 1
+3 0 doc1964 1
+3 0 doc10597 1
+3 0 doc21023 1
+3 0 doc19057 1
+3 0 doc14948 1
+3 0 doc9692 1
+
+
+4 0 doc2534 1
+4 0 doc21388 1
+4 0 doc20923 1
+4 0 doc11547 1
+4 0 doc19755 1
+4 0 doc3793 1
+4 0 doc6714 1
+4 0 doc12722 1
+4 0 doc5552 1
+4 0 doc6810 1
+4 0 doc16953 1
+4 0 doc2527 1
+4 0 doc5361 1
+4 0 doc12353 1
+4 0 doc7308 1
+4 0 doc3836 1
+4 0 doc2293 1
+4 0 doc7348 1
+4 0 doc17119 1
+4 0 doc19331 1
+4 0 doc3411 1
+4 0 doc14643 1
+4 0 doc9058 1
+4 0 doc11099 1
+4 0 doc12485 1
+4 0 doc16432 1
+4 0 doc10047 1
+4 0 doc13788 1
+4 0 doc117 1
+4 0 doc638 1
+
+
+
+5 0 doc169 1
+5 0 doc13181 1
+5 0 doc4350 1
+5 0 doc10242 1
+5 0 doc955 1
+5 0 doc5389 1
+5 0 doc17122 1
+5 0 doc17417 1
+5 0 doc12199 1
+5 0 doc6918 1
+5 0 doc3857 1
+5 0 doc2981 1
+5 0 doc10639 1
+5 0 doc10478 1
+5 0 doc8573 1
+5 0 doc9197 1
+5 0 doc9298 1
+5 0 doc2492 1
+5 0 doc10262 1
+5 0 doc5180 1
+5 0 doc11758 1
+5 0 doc4065 1
+5 0 doc9124 1
+5 0 doc11528 1
+5 0 doc18879 1
+5 0 doc17864 1
+5 0 doc3204 1
+5 0 doc12157 1
+5 0 doc4496 1
+5 0 doc20190 1
+
+
+
+6 0 doc9507 1
+6 0 doc15630 1
+6 0 doc8469 1
+6 0 doc11918 1
+6 0 doc20482 1
+6 0 doc20158 1
+6 0 doc19831 1
+6 0 doc8296 1
+6 0 doc8930 1
+6 0 doc16460 1
+6 0 doc2577 1
+6 0 doc15476 1
+6 0 doc1767 1
+6 0 doc689 1
+6 0 doc16606 1
+6 0 doc6149 1
+6 0 doc18691 1
+6 0 doc2208 1
+6 0 doc3592 1
+6 0 doc11199 1
+6 0 doc16329 1
+6 0 doc6007 1
+6 0 doc15231 1
+6 0 doc20622 1
+6 0 doc21468 1
+6 0 doc12230 1
+6 0 doc5723 1
+6 0 doc8120 1
+6 0 doc8668 1
+6 0 doc303 1
+
+
+
+
+7 0 doc7728 1
+7 0 doc7693 1
+7 0 doc21088 1
+7 0 doc5017 1
+7 0 doc10807 1
+7 0 doc16204 1
+7 0 doc2233 1
+7 0 doc3632 1
+7 0 doc4719 1
+7 0 doc6477 1
+7 0 doc6502 1
+7 0 doc6709 1
+7 0 doc7710 1
+7 0 doc9193 1
+7 0 doc9309 1
+7 0 doc9789 1
+7 0 doc10971 1
+7 0 doc18059 1
+7 0 doc19906 1
+7 0 doc20089 1
+7 0 doc20102 1
+7 0 doc21040 1
+7 0 doc21153 1
+7 0 doc9147 1
+7 0 doc9930 1
+7 0 doc19763 1
+7 0 doc1559 1
+7 0 doc21248 1
+7 0 doc17945 1
+7 0 doc526 1
+
+
+# --- m==0: avg_precision and recall are hurt, by marking fake docs as relevant
+
+8 0 fakedoc1 1
+8 0 fakedoc2 1
+8 0 fakedoc3 1
+8 0 fakedoc4 1
+
+8 0 doc16299 1
+8 0 doc1662 1
+8 0 doc4585 1
+8 0 doc12315 1
+8 0 doc16266 1
+8 0 doc13136 1
+8 0 doc19212 1
+8 0 doc7086 1
+8 0 doc7062 1
+8 0 doc6134 1
+8 0 doc13953 1
+8 0 doc16264 1
+8 0 doc2494 1
+8 0 doc10636 1
+8 0 doc10894 1
+8 0 doc6844 1
+8 0 doc674 1
+8 0 doc13520 1
+8 0 doc344 1
+8 0 doc2896 1
+8 0 doc11871 1
+8 0 doc1862 1
+8 0 doc16728 1
+8 0 doc10308 1
+8 0 doc2227 1
+8 0 doc13167 1
+8 0 doc20607 1
+8 0 doc9670 1
+8 0 doc1566 1
+8 0 doc17885 1
+
+
+# ---- m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
+
+
+9 0 doc1990 0
+9 0 doc9342 1
+9 0 doc19427 1
+9 0 doc12432 0
+9 0 doc13480 1
+9 0 doc3322 1
+9 0 doc16044 1
+9 0 doc266 0
+9 0 doc3437 1
+9 0 doc5370 1
+9 0 doc10314 1
+9 0 doc4892 1
+9 0 doc5763 0
+9 0 doc14045 1
+9 0 doc1090 1
+9 0 doc7437 1
+9 0 doc5822 1
+9 0 doc4285 1
+9 0 doc17119 1
+9 0 doc21001 1
+9 0 doc4337 1
+9 0 doc5967 1
+9 0 doc10214 1
+9 0 doc12001 1
+9 0 doc18553 1
+9 0 doc12116 1
+9 0 doc5064 1
+9 0 doc5018 1
+9 0 doc5037 1
+9 0 doc8025 1
+
+
+# ---- m==2: all precision, precision_at_n and recall are hurt.
+
+10 0 fakedoc1 1
+10 0 fakedoc2 1
+10 0 fakedoc3 1
+10 0 fakedoc4 1
+
+10 0 doc17218 0
+10 0 doc10270 0
+10 0 doc5958 0
+10 0 doc19943 0
+10 0 doc6510 1
+10 0 doc16087 1
+10 0 doc14893 1
+10 0 doc8933 1
+10 0 doc4354 1
+10 0 doc16729 1
+10 0 doc16761 1
+10 0 doc6964 1
+10 0 doc16743 1
+10 0 doc7357 1
+10 0 doc2534 1
+10 0 doc18321 1
+10 0 doc18497 1
+10 0 doc11214 1
+10 0 doc11819 1
+10 0 doc10818 1
+10 0 doc15769 1
+10 0 doc5348 1
+10 0 doc14948 1
+10 0 doc7891 1
+10 0 doc9897 1
+10 0 doc15559 1
+10 0 doc14935 1
+10 0 doc14954 1
+10 0 doc6621 1
+10 0 doc6930 1
+
+
+11 0 doc11943 1
+11 0 doc286 1
+11 0 doc1574 1
+11 0 doc17916 1
+11 0 doc17918 1
+11 0 doc19213 1
+11 0 doc9337 1
+11 0 doc8593 1
+11 0 doc8800 1
+11 0 doc18580 1
+11 0 doc209 1
+11 0 doc1893 1
+11 0 doc11189 1
+11 0 doc17702 1
+11 0 doc10180 1
+11 0 doc11869 1
+11 0 doc9705 1
+11 0 doc8715 1
+11 0 doc12753 1
+11 0 doc10195 1
+11 0 doc3552 1
+11 0 doc16030 1
+11 0 doc4623 1
+11 0 doc3188 1
+11 0 doc8735 1
+11 0 doc151 1
+11 0 doc5792 1
+11 0 doc5194 1
+11 0 doc3393 1
+11 0 doc19027 1
+
+
+
+12 0 doc18198 1
+12 0 doc2444 1
+12 0 doc4305 1
+12 0 doc6544 1
+12 0 doc11639 1
+12 0 doc10640 1
+12 0 doc12192 1
+12 0 doc128 1
+12 0 doc10760 1
+12 0 doc10881 1
+12 0 doc2698 1
+12 0 doc3552 1
+12 0 doc20524 1
+12 0 doc1884 1
+12 0 doc9187 1
+12 0 doc3131 1
+12 0 doc2911 1
+12 0 doc2589 1
+12 0 doc3747 1
+12 0 doc3813 1
+12 0 doc5222 1
+12 0 doc6023 1
+12 0 doc6624 1
+12 0 doc7655 1
+12 0 doc9205 1
+12 0 doc12062 1
+12 0 doc15504 1
+12 0 doc13625 1
+12 0 doc18704 1
+12 0 doc2277 1
+
+
+
+13 0 doc4948 1
+13 0 doc21565 1
+13 0 doc17135 1
+13 0 doc1866 1
+13 0 doc13989 1
+13 0 doc5605 1
+13 0 doc13431 1
+13 0 doc2100 1
+13 0 doc16347 1
+13 0 doc16894 1
+13 0 doc6764 1
+13 0 doc8554 1
+13 0 doc8695 1
+13 0 doc8977 1
+13 0 doc19478 1
+13 0 doc14595 1
+13 0 doc2408 1
+13 0 doc2592 1
+13 0 doc10947 1
+13 0 doc15794 1
+13 0 doc5236 1
+13 0 doc14847 1
+13 0 doc3980 1
+13 0 doc1844 1
+13 0 doc42 1
+13 0 doc7783 1
+13 0 doc4557 1
+13 0 doc16423 1
+13 0 doc17170 1
+13 0 doc5822 1
+
+
+
+14 0 doc17172 1
+14 0 doc17210 1
+14 0 doc5044 1
+14 0 doc4627 1
+14 0 doc4683 1
+14 0 doc15126 1
+14 0 doc4538 1
+14 0 doc273 1
+14 0 doc19585 1
+14 0 doc16078 1
+14 0 doc4529 1
+14 0 doc4186 1
+14 0 doc12961 1
+14 0 doc19217 1
+14 0 doc5670 1
+14 0 doc1699 1
+14 0 doc4716 1
+14 0 doc12644 1
+14 0 doc18387 1
+14 0 doc336 1
+14 0 doc16130 1
+14 0 doc18718 1
+14 0 doc12527 1
+14 0 doc11797 1
+14 0 doc11831 1
+14 0 doc7538 1
+14 0 doc17259 1
+14 0 doc18724 1
+14 0 doc19330 1
+14 0 doc19206 1
+
+
+
+15 0 doc12198 1
+15 0 doc20371 1
+15 0 doc2947 1
+15 0 doc10750 1
+15 0 doc7239 1
+15 0 doc14189 1
+15 0 doc19474 1
+15 0 doc14776 1
+15 0 doc21270 1
+15 0 doc6387 1
+15 0 doc12908 1
+15 0 doc9573 1
+15 0 doc17102 1
+15 0 doc21482 1
+15 0 doc6524 1
+15 0 doc18034 1
+15 0 doc1358 1
+15 0 doc13147 1
+15 0 doc17731 1
+15 0 doc12890 1
+15 0 doc20887 1
+15 0 doc19508 1
+15 0 doc18498 1
+15 0 doc20642 1
+15 0 doc19878 1
+15 0 doc6556 1
+15 0 doc10272 1
+15 0 doc5720 1
+15 0 doc17578 1
+15 0 doc17164 1
+
+
+# --- m==0: avg_precision and recall are hurt, by marking fake docs as relevant
+
+16 0 fakedoc1 1
+16 0 fakedoc2 1
+16 0 fakedoc3 1
+16 0 fakedoc4 1
+
+16 0 doc4043 1
+16 0 doc14985 1
+16 0 doc15370 1
+16 0 doc15426 1
+16 0 doc1702 1
+16 0 doc3062 1
+16 0 doc16134 1
+16 0 doc15037 1
+16 0 doc8224 1
+16 0 doc5044 1
+16 0 doc8545 1
+16 0 doc7228 1
+16 0 doc12686 1
+16 0 doc16609 1
+16 0 doc13161 1
+16 0 doc3446 1
+16 0 doc16493 1
+16 0 doc19297 1
+16 0 doc13619 1
+16 0 doc3281 1
+16 0 doc15499 1
+16 0 doc7373 1
+16 0 doc9064 1
+16 0 doc1710 1
+16 0 doc15411 1
+16 0 doc10890 1
+16 0 doc3166 1
+16 0 doc17894 1
+16 0 doc4560 1
+16 0 doc12766 1
+
+
+# --- m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
+
+17 0 doc3117 0
+17 0 doc7477 0
+17 0 doc7569 0
+17 0 doc20667 0
+17 0 doc20260 1
+17 0 doc17355 1
+17 0 doc11021 1
+17 0 doc20934 1
+17 0 doc552 1
+17 0 doc20856 1
+17 0 doc3524 1
+17 0 doc17343 1
+17 0 doc21055 1
+17 0 doc19032 1
+17 0 doc19786 1
+17 0 doc9281 1
+17 0 doc1695 1
+17 0 doc15940 1
+17 0 doc9215 1
+17 0 doc8335 1
+17 0 doc20936 1
+17 0 doc6914 1
+17 0 doc12122 1
+17 0 doc6618 1
+17 0 doc5049 1
+17 0 doc450 1
+17 0 doc19206 1
+17 0 doc18823 1
+17 0 doc5307 1
+17 0 doc17295 1
+
+
+# ---- m==2: all precision, precision_at_n and recall are hurt.
+
+18 0 fakedoc1 1
+18 0 fakedoc2 1
+18 0 fakedoc3 1
+18 0 fakedoc4 1
+
+18 0 doc8064 0
+18 0 doc18142 0
+18 0 doc19383 0
+18 0 doc21151 0
+18 0 doc4665 1
+18 0 doc2897 1
+18 0 doc6878 1
+18 0 doc14507 1
+18 0 doc2976 1
+18 0 doc11757 1
+18 0 doc12625 1
+18 0 doc14908 1
+18 0 doc12790 1
+18 0 doc17915 1
+18 0 doc11804 1
+18 0 doc12935 1
+18 0 doc8225 1
+18 0 doc18011 1
+18 0 doc10493 1
+18 0 doc17922 1
+18 0 doc1902 1
+18 0 doc14049 1
+18 0 doc1334 1
+18 0 doc1168 1
+18 0 doc4859 1
+18 0 doc7124 1
+18 0 doc9692 1
+18 0 doc18402 1
+18 0 doc9089 1
+18 0 doc15375 1
+
+
+19 0 doc5267 1
+19 0 doc2310 1
+19 0 doc11435 1
+19 0 doc15666 1
+19 0 doc12733 1
+19 0 doc7925 1
+19 0 doc2444 1
+19 0 doc4900 1
+19 0 doc10803 1
+19 0 doc8869 1
+19 0 doc5051 1
+19 0 doc9163 1
+19 0 doc529 1
+19 0 doc19546 1
+19 0 doc18561 1
+19 0 doc10634 1
+19 0 doc3979 1
+19 0 doc8833 1
+19 0 doc7652 1
+19 0 doc4804 1
+19 0 doc12616 1
+19 0 doc8419 1
+19 0 doc9431 1
+19 0 doc16235 1
+19 0 doc732 1
+19 0 doc2515 1
+19 0 doc7194 1
+19 0 doc16301 1
+19 0 doc4494 1
+19 0 doc4496 1
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Quality/trecTopics.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Quality/trecTopics.txt b/src/Lucene.Net.Tests.Benchmark/Quality/trecTopics.txt
new file mode 100644
index 0000000..2f3ada2
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Quality/trecTopics.txt
@@ -0,0 +1,287 @@
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -----------------------------------------------------------------------
+
+# ------------------------------------------------------------
+# This file was created using utils.QualityQueriesFinder.
+# See also TrecQRels.txt.
+# ------------------------------------------------------------
+
+<top>
+<num> Number: 0
+
+<title> statement months total 1987
+
+<desc> Description:
+Topic 0 Description Line 1
+Topic 0 Description Line 2
+
+<narr> Narrative:
+Topic 0 Narrative Line 1
+Topic 0 Narrative Line 2
+
+</top>
+
+<top>
+<num> Number: 1
+
+<title> agreed 15 against five
+
+<desc> Description:
+Topic 1 Description Line 1
+Topic 1 Description Line 2
+
+<narr> Narrative:
+Topic 1 Narrative Line 1
+Topic 1 Narrative Line 2
+
+</top>
+
+<top>
+<num> Number: 2
+
+<title> nine only month international
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 3
+
+<title> finance any 10 government
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 4
+
+<title> issue next years all
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 5
+
+<title> who major ltd today
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 6
+
+<title> business revs securities per
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 7
+
+<title> quarter time note sales
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 8
+
+<title> february earlier loss group
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 9
+
+<title> out end made some
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 10
+
+<title> spokesman financial 30 expected
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 11
+
+<title> 1985 now prices due
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 12
+
+<title> before board record could
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 13
+
+<title> pay debt because trade
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 14
+
+<title> meeting increase four price
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 15
+
+<title> chairman rate six interest
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 16
+
+<title> since current between agreement
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 17
+
+<title> oil we when president
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 18
+
+<title> capital through foreign added
+
+<desc> Description:
+
+
+<narr> Narrative:
+
+
+</top>
+
+<top>
+<num> Number: 19
+
+<title> 20 while common week
+
+<desc> Description:
+Topic 19 Description Line 1
+Topic 19 Description Line 2
+
+<narr> Narrative:
+Topic 19 Narrative Line 1
+Topic 19 Narrative Line 2
+
+</top>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Support/TestApiConsistency.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Support/TestApiConsistency.cs b/src/Lucene.Net.Tests.Benchmark/Support/TestApiConsistency.cs
new file mode 100644
index 0000000..93c424e
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Support/TestApiConsistency.cs
@@ -0,0 +1,150 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+using Lucene.Net.Attributes;
+using Lucene.Net.Support;
+using NUnit.Framework;
+using System;
+
+namespace Lucene.Net.Benchmarks.Support
+{
+ /// <summary>
+ /// LUCENENET specific tests for ensuring API conventions are followed
+ /// </summary>
+ public class TestApiConsistency : ApiScanTestBase
+ {
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestProtectedFieldNames(Type typeFromTargetAssembly)
+ {
+ base.TestProtectedFieldNames(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestPrivateFieldNames(Type typeFromTargetAssembly)
+ {
+ base.TestPrivateFieldNames(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestPublicFields(Type typeFromTargetAssembly)
+ {
+ base.TestPublicFields(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestMethodParameterNames(Type typeFromTargetAssembly)
+ {
+ base.TestMethodParameterNames(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestInterfaceNames(Type typeFromTargetAssembly)
+ {
+ base.TestInterfaceNames(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestClassNames(Type typeFromTargetAssembly)
+ {
+ base.TestClassNames(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForPropertiesWithNoGetter(Type typeFromTargetAssembly)
+ {
+ base.TestForPropertiesWithNoGetter(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForPropertiesThatReturnArray(Type typeFromTargetAssembly)
+ {
+ base.TestForPropertiesThatReturnArray(typeFromTargetAssembly);
+ }
+
+#if !NETSTANDARD
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForMethodsThatReturnWritableArray(Type typeFromTargetAssembly)
+ {
+ base.TestForMethodsThatReturnWritableArray(typeFromTargetAssembly);
+ }
+#endif
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForPublicMembersContainingComparer(Type typeFromTargetAssembly)
+ {
+ base.TestForPublicMembersContainingComparer(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForPublicMembersNamedSize(Type typeFromTargetAssembly)
+ {
+ base.TestForPublicMembersNamedSize(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForPublicMembersContainingNonNetNumeric(Type typeFromTargetAssembly)
+ {
+ base.TestForPublicMembersContainingNonNetNumeric(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForTypesContainingNonNetNumeric(Type typeFromTargetAssembly)
+ {
+ base.TestForTypesContainingNonNetNumeric(typeFromTargetAssembly);
+ }
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForPublicMembersWithNullableEnum(Type typeFromTargetAssembly)
+ {
+ base.TestForPublicMembersWithNullableEnum(typeFromTargetAssembly);
+ }
+
+ // LUCENENET NOTE: This test is only for identifying members who were changed from
+ // ICollection, IList or ISet to IEnumerable during the port (that should be changed back)
+ //[Test, LuceneNetSpecific]
+ //[TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ //public override void TestForMembersAcceptingOrReturningIEnumerable(Type typeFromTargetAssembly)
+ //{
+ // base.TestForMembersAcceptingOrReturningIEnumerable(typeFromTargetAssembly);
+ //}
+
+ [Test, LuceneNetSpecific]
+ [TestCase(typeof(Lucene.Net.Benchmarks.Constants))]
+ public override void TestForMembersAcceptingOrReturningListOrDictionary(Type typeFromTargetAssembly)
+ {
+ base.TestForMembersAcceptingOrReturningListOrDictionary(typeFromTargetAssembly);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/Support/TestExceptionSerialization.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Support/TestExceptionSerialization.cs b/src/Lucene.Net.Tests.Benchmark/Support/TestExceptionSerialization.cs
new file mode 100644
index 0000000..fdd7b5b
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Support/TestExceptionSerialization.cs
@@ -0,0 +1,54 @@
+#if FEATURE_SERIALIZABLE
+using Lucene.Net.Attributes;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Lucene.Net.Support
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ [TestFixture]
+ public class TestExceptionSerialization : ExceptionSerializationTestBase
+ {
+ public static IEnumerable<object> ExceptionTestData
+ {
+ get
+ {
+ var exceptionTypes = typeof(Lucene.Net.Benchmarks.Constants).Assembly.GetTypes().Where(t => typeof(Exception).IsAssignableFrom(t)).Cast<object>();
+
+ // If the assembly has no exceptions, just provide Exception so the test will pass
+ if (!exceptionTypes.Any())
+ {
+ return new Type[] { typeof(Exception) };
+ }
+
+ return exceptionTypes;
+ }
+ }
+
+ [Test, LuceneNetSpecific]
+ public void AllExceptionsInLuceneNamespaceCanSerialize([ValueSource("ExceptionTestData")]Type luceneException)
+ {
+ var instance = TryInstantiate(luceneException);
+ Assert.That(TypeCanSerialize(instance), string.Format("Unable to serialize {0}", luceneException.FullName));
+ }
+ }
+}
+#endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/project.json b/src/Lucene.Net.Tests.Benchmark/project.json
new file mode 100644
index 0000000..5babcd7
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/project.json
@@ -0,0 +1,56 @@
+{
+ "version": "4.8.0",
+ "title": "Lucene.Net.Tests.Benchmark",
+ "buildOptions": {
+ "compile": {
+ "includeFiles": [ "../CommonAssemblyInfo.cs" ]
+ },
+ "embed": {
+ "includeFiles": [
+ "ByTask/Feeds/trecdocs.zip",
+ "ByTask/conf.zip",
+ "ByTask/reuters.first20.lines.txt",
+ "ByTask/test-mapping-ISOLatin1Accent-partial.txt",
+ "Quality/reuters.578.lines.txt.bz2",
+ "Quality/trecQRels.txt",
+ "Quality/trecTopics.txt"
+ ]
+ }
+ },
+ "dependencies": {
+ "dotnet-test-nunit-teamcity": "3.4.0-beta-3",
+ "icu.net": "54.1.1-alpha",
+ "Lucene.Net.Analysis.Common": "4.8.0",
+ "Lucene.Net.Benchmark": "4.8.0",
+ "Lucene.Net.Facet": "4.8.0",
+ "Lucene.Net.Highlighter": "4.8.0",
+ "Lucene.Net.ICU": "4.8.0",
+ "Lucene.Net.TestFramework": "4.8.0",
+ "NUnit": "3.5.0"
+ },
+ "testRunner": "nunit-teamcity",
+ "frameworks": {
+ "netcoreapp1.0": {
+ "imports": "dnxcore50",
+ "buildOptions": {
+ "debugType": "portable",
+ "define": [ "NETSTANDARD" ]
+ }
+ },
+ "net451": {
+ "buildOptions": {
+ "debugType": "full",
+ "define": [
+ "FEATURE_SERIALIZABLE",
+ "FEATURE_THREAD_PRIORITY",
+ "FEATURE_THREADINTERRUPTEDEXCEPTION"
+ ]
+ }
+ }
+ },
+
+ "runtimes": {
+ "win7-x86": {},
+ "win7-x64": {}
+ }
+}
[28/33] lucenenet git commit: Changed to a new MyGet feed named
lucene-icu-dotnet and added build latest changes
Posted by ni...@apache.org.
Changed to a new MyGet feed named lucene-icu-dotnet and added build latest changes
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/acc82da4
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/acc82da4
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/acc82da4
Branch: refs/heads/master
Commit: acc82da4d1bbf0826f0c89a1dee653edaaf77486
Parents: 56a18c3
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Aug 4 18:58:11 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 19:33:36 2017 +0700
----------------------------------------------------------------------
NuGet.config | 2 +-
.../Lucene.Net.Analysis.SmartCn.project.json | 2 +-
src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json | 2 +-
src/Lucene.Net.ICU/Lucene.Net.ICU.project.json | 2 +-
src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json | 2 +-
.../Lucene.Net.Tests.Benchmark.project.json | 2 +-
src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/NuGet.config
----------------------------------------------------------------------
diff --git a/NuGet.config b/NuGet.config
index 64c38ad..021fe6f 100644
--- a/NuGet.config
+++ b/NuGet.config
@@ -22,7 +22,7 @@
<configuration>
<packageSources>
<clear />
- <add key="icunet" value="https://www.myget.org/F/icu-dotnet/api/v2" />
+ <add key="icu-dotnet" value="https://www.myget.org/F/lucene-icu-dotnet/api/v2" />
<add key="dotnet-cat" value="https://www.myget.org/F/dotnetcat/api/v2" />
<add key="nugetorg" value="https://www.nuget.org/api/v2" />
</packageSources>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json b/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
index 74d9d80..1551a49 100644
--- a/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
+++ b/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu.net": "54.1.1-alpha"
+ "icu-dotnet": "2.2.0-netcore0078"
},
"frameworks": {
"net451": {}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
index f764f6a..83d487f 100644
--- a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu.net": "54.1.1-alpha",
+ "icu-dotnet": "2.2.0-netcore0078",
"SharpZipLib": "0.86.0",
"Spatial4n.Core": "0.4.1-beta00003"
},
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json b/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
index af28fc8..496bea5 100644
--- a/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
+++ b/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu.net": "54.1.1-alpha"
+ "icu-dotnet": "2.2.0-netcore0078"
},
"frameworks": {
"net451": {}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json b/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
index 36faa44..b217af9 100644
--- a/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
+++ b/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu.net": "54.1.1-alpha",
+ "icu-dotnet": "2.2.0-netcore0078",
"NUnit": "3.5.0"
},
"frameworks": {
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
index 41388a7..ec41dd3 100644
--- a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
+++ b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu.net": "54.1.1-alpha",
+ "icu-dotnet": "2.2.0-netcore0078",
"NUnit": "3.5.0",
"SharpZipLib": "0.86.0"
},
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/acc82da4/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json b/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
index 2a96d5b..fd1d44f 100644
--- a/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
+++ b/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
@@ -4,7 +4,7 @@
},
"dependencies": {
"NUnit": "3.5.0",
- "icu.net": "54.1.1-alpha"
+ "icu-dotnet": "2.2.0-netcore0078"
},
"frameworks": {
"net451": {}
[25/33] lucenenet git commit: Added Lucene.Net.Benchmark + the
Collator functionality in Lucene.Net.ICU to .NET Standard and fixed bugs
(still have 4 failing tests).
Posted by ni...@apache.org.
Added Lucene.Net.Benchmark + the Collator functionality in Lucene.Net.ICU to .NET Standard and fixed bugs (still have 4 failing tests).
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/4cb35e92
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/4cb35e92
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/4cb35e92
Branch: refs/heads/master
Commit: 4cb35e92cdb269f76b1ad3577f8e8d830542cebd
Parents: 198e586
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Aug 4 14:28:16 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 19:33:32 2017 +0700
----------------------------------------------------------------------
Lucene.Net.Portable.sln | 20 ++++++++++
.../Collation/ICUCollationDocValuesField.cs | 3 ++
.../Collation/ICUCollationKeyFilter.cs | 3 ++
.../ICUCollatedTermAttributeImpl.cs | 3 ++
.../ByTask/Feeds/EnwikiContentSource.cs | 28 ++++++-------
.../ByTask/Tasks/AnalyzerFactoryTask.cs | 13 +++---
.../ByTask/Tasks/CreateIndexTask.cs | 14 +++++++
.../ByTask/Tasks/TaskSequence.cs | 2 +
.../ByTask/Tasks/WriteLineDocTask.cs | 2 +-
.../ByTask/Utils/Algorithm.cs | 2 +-
.../Lucene.Net.Benchmark.csproj | 2 +-
.../Lucene.Net.Benchmark.xproj | 39 ++++++++++++++++++
.../Support/TagSoup/ElementType.cs | 7 ++--
.../Support/TagSoup/PYXWriter.cs | 4 +-
.../Support/TagSoup/Parser.cs | 2 +-
src/Lucene.Net.Benchmark/project.json | 27 +++++++------
src/Lucene.Net.ICU/project.json | 9 ++++-
.../Collation/TestICUCollationKeyAnalyzer.cs | 3 ++
.../ByTask/Feeds/DocMakerTest.cs | 4 +-
.../ByTask/Feeds/EnwikiContentSourceTest.cs | 2 +-
.../Lucene.Net.Tests.Benchmark.xproj | 42 ++++++++++++++++++++
.../Properties/AssemblyInfo.cs | 17 +-------
src/Lucene.Net.Tests.Benchmark/project.json | 18 ++++-----
src/Lucene.Net.Tests.ICU/project.json | 8 ++++
24 files changed, 203 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/Lucene.Net.Portable.sln
----------------------------------------------------------------------
diff --git a/Lucene.Net.Portable.sln b/Lucene.Net.Portable.sln
index e94a262..5dbbd51 100644
--- a/Lucene.Net.Portable.sln
+++ b/Lucene.Net.Portable.sln
@@ -107,6 +107,10 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Lucene.Net.Analysis.Kuromoj
EndProject
Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Lucene.Net.Tests.Analysis.Kuromoji", "src\Lucene.Net.Tests.Analysis.Kuromoji\Lucene.Net.Tests.Analysis.Kuromoji.xproj", "{F82F0F31-09E7-48FB-B5FF-F3A84627A307}"
EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Lucene.Net.Benchmark", "src\Lucene.Net.Benchmark\Lucene.Net.Benchmark.xproj", "{AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Lucene.Net.Tests.Benchmark", "src\Lucene.Net.Tests.Benchmark\Lucene.Net.Tests.Benchmark.xproj", "{5356412F-8FC5-41D3-83C3-807740B06748}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -491,6 +495,22 @@ Global
{F82F0F31-09E7-48FB-B5FF-F3A84627A307}.Release|Any CPU.Build.0 = Release|Any CPU
{F82F0F31-09E7-48FB-B5FF-F3A84627A307}.Release|x86.ActiveCfg = Release|Any CPU
{F82F0F31-09E7-48FB-B5FF-F3A84627A307}.Release|x86.Build.0 = Release|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Debug|x86.Build.0 = Debug|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Release|Any CPU.Build.0 = Release|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Release|x86.ActiveCfg = Release|Any CPU
+ {AFBD39F9-4C9F-4CC2-BF2F-296D7993CA32}.Release|x86.Build.0 = Release|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Debug|x86.Build.0 = Debug|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Release|Any CPU.Build.0 = Release|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Release|x86.ActiveCfg = Release|Any CPU
+ {5356412F-8FC5-41D3-83C3-807740B06748}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationDocValuesField.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationDocValuesField.cs b/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationDocValuesField.cs
index bddc095..4217ba4 100644
--- a/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationDocValuesField.cs
+++ b/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationDocValuesField.cs
@@ -1,4 +1,7 @@
using Icu.Collation;
+#if NETSTANDARD
+using Icu.ObjectModel; // For SortKey
+#endif
using Lucene.Net.Documents;
using Lucene.Net.Support;
using Lucene.Net.Util;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilter.cs b/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilter.cs
index e6c595a..f84acfe 100644
--- a/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilter.cs
+++ b/src/Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilter.cs
@@ -1,4 +1,7 @@
using Icu.Collation;
+#if NETSTANDARD
+using Icu.ObjectModel; // For SortKey
+#endif
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Support;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Analysis.ICU/Collation/TokenAttributes/ICUCollatedTermAttributeImpl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.ICU/Collation/TokenAttributes/ICUCollatedTermAttributeImpl.cs b/src/Lucene.Net.Analysis.ICU/Collation/TokenAttributes/ICUCollatedTermAttributeImpl.cs
index ac1187e..1fdca92 100644
--- a/src/Lucene.Net.Analysis.ICU/Collation/TokenAttributes/ICUCollatedTermAttributeImpl.cs
+++ b/src/Lucene.Net.Analysis.ICU/Collation/TokenAttributes/ICUCollatedTermAttributeImpl.cs
@@ -1,4 +1,7 @@
using Icu.Collation;
+#if NETSTANDARD
+using Icu.ObjectModel; // For SortKey
+#endif
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Support;
using Lucene.Net.Util;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
index 6870b3b..c310c2d 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
@@ -77,14 +77,14 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
{
while (tuple == null && nmde == null && !threadDone && !stopped)
{
- try
- {
+ //try
+ //{
Monitor.Wait(this);
- }
- catch (ThreadInterruptedException ie)
- {
- throw new ThreadInterruptedException(ie.ToString(), ie);
- }
+ //}
+ //catch (ThreadInterruptedException ie)
+ //{
+ // throw new ThreadInterruptedException(ie.ToString(), ie);
+ //}
}
if (tuple != null)
{
@@ -148,14 +148,14 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
{
while (tuple != null && !stopped)
{
- try
- {
+ //try
+ //{
Monitor.Wait(this); //wait();
- }
- catch (ThreadInterruptedException ie)
- {
- throw new ThreadInterruptedException(ie.ToString(), ie);
- }
+ //}
+ //catch (ThreadInterruptedException ie)
+ //{
+ // throw new ThreadInterruptedException(ie.ToString(), ie);
+ //}
}
tuple = tmpTuple;
Monitor.Pulse(this); //notify();
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
index 56b0114..61d1832 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
@@ -6,6 +6,7 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
+using System.Reflection;
using System.Text.RegularExpressions;
namespace Lucene.Net.Benchmarks.ByTask.Tasks
@@ -484,15 +485,15 @@ namespace Lucene.Net.Benchmarks.ByTask.Tasks
DirectoryInfo baseDir = new DirectoryInfo(RunData.Config.Get("work.dir", "work"));
((IResourceLoaderAware)instance).Inform(new FilesystemResourceLoader(baseDir));
}
- if (typeof(CharFilterFactory).IsAssignableFrom(clazz))
+ if (typeof(CharFilterFactory).GetTypeInfo().IsAssignableFrom(clazz))
{
charFilterFactories.Add((CharFilterFactory)instance);
}
- else if (typeof(TokenizerFactory).IsAssignableFrom(clazz))
+ else if (typeof(TokenizerFactory).GetTypeInfo().IsAssignableFrom(clazz))
{
tokenizerFactory = (TokenizerFactory)instance;
}
- else if (typeof(TokenFilterFactory).IsAssignableFrom(clazz))
+ else if (typeof(TokenFilterFactory).GetTypeInfo().IsAssignableFrom(clazz))
{
tokenFilterFactories.Add((TokenFilterFactory)instance);
}
@@ -549,15 +550,15 @@ namespace Lucene.Net.Benchmarks.ByTask.Tasks
}
// No dot - use analysis SPI lookup
string analysisComponentName = ANALYSIS_COMPONENT_SUFFIX_PATTERN.Replace(className, "", 1);
- if (typeof(CharFilterFactory).IsAssignableFrom(expectedType))
+ if (typeof(CharFilterFactory).GetTypeInfo().IsAssignableFrom(expectedType))
{
return CharFilterFactory.LookupClass(analysisComponentName);
}
- else if (typeof(TokenizerFactory).IsAssignableFrom(expectedType))
+ else if (typeof(TokenizerFactory).GetTypeInfo().IsAssignableFrom(expectedType))
{
return TokenizerFactory.LookupClass(analysisComponentName);
}
- else if (typeof(TokenFilterFactory).IsAssignableFrom(expectedType))
+ else if (typeof(TokenFilterFactory).GetTypeInfo().IsAssignableFrom(expectedType))
{
return TokenFilterFactory.LookupClass(analysisComponentName);
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
index 046ed25..9705c7a 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
@@ -110,6 +110,14 @@ namespace Lucene.Net.Benchmarks.ByTask.Tasks
string mergeScheduler = config.Get("merge.scheduler",
"Lucene.Net.Index.ConcurrentMergeScheduler, Lucene.Net");
+#if !FEATURE_CONCURRENTMERGESCHEDULER
+ // LUCENENET specific - hack to get our TaskMergeScheduler
+ // when a ConcurrentMergeScheduler is requested.
+ if (mergeScheduler.Contains(".ConcurrentMergeScheduler,"))
+ {
+ mergeScheduler = "Lucene.Net.Index.TaskMergeScheduler, Lucene.Net";
+ }
+#endif
Type mergeSchedulerType = Type.GetType(mergeScheduler);
if (mergeSchedulerType == null)
{
@@ -132,9 +140,15 @@ namespace Lucene.Net.Benchmarks.ByTask.Tasks
if (mergeScheduler.Equals("Lucene.Net.Index.ConcurrentMergeScheduler"))
{
+#if FEATURE_CONCURRENTMERGESCHEDULER
ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)iwConf.MergeScheduler;
int maxThreadCount = config.Get("concurrent.merge.scheduler.max.thread.count", ConcurrentMergeScheduler.DEFAULT_MAX_THREAD_COUNT);
int maxMergeCount = config.Get("concurrent.merge.scheduler.max.merge.count", ConcurrentMergeScheduler.DEFAULT_MAX_MERGE_COUNT);
+#else
+ TaskMergeScheduler cms = (TaskMergeScheduler)iwConf.MergeScheduler;
+ int maxThreadCount = config.Get("concurrent.merge.scheduler.max.thread.count", 1);
+ int maxMergeCount = config.Get("concurrent.merge.scheduler.max.merge.count", 2);
+#endif
cms.SetMaxMergesAndThreads(maxMergeCount, maxThreadCount);
}
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
index 9be1b5d..12d91c0 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
@@ -218,7 +218,9 @@ namespace Lucene.Net.Benchmarks.ByTask.Tasks
bgTasks = new List<RunBackgroundTask>();
}
RunBackgroundTask bgTask = new RunBackgroundTask(task, letChildReport);
+#if FEATURE_THREAD_PRIORITY
bgTask.Priority = (task.BackgroundDeltaPriority + Thread.CurrentThread.Priority);
+#endif
bgTask.Start();
bgTasks.Add(bgTask);
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
index f9e7546..269f98f 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
@@ -127,7 +127,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Tasks
else
{
checkSufficientFields = true;
- HashSet<string> sf = new HashSet<string>(suff.Split(new char[] { ',' }).TrimEnd());
+ HashSet<string> sf = new HashSet<string>(suff.Split(',').TrimEnd());
for (int i = 0; i < fieldsToWrite.Length; i++)
{
if (sf.Contains(fieldsToWrite[i]))
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
index 7e6523e..cc54e64 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
@@ -397,7 +397,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Utils
private Type LoadType(string assemblyName, string typeName)
{
- return Assembly.Load(assemblyName).DefinedTypes.FirstOrDefault(t => t.Name == typeName);
+ return Assembly.Load(new AssemblyName(assemblyName)).GetTypes().FirstOrDefault(t => t.Name == typeName);
}
public override string ToString()
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
index 2645226..59fc648 100644
--- a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
@@ -32,7 +32,7 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup>
- <DefineConstants>$(DefineConstants);FEATURE_SERIALIZABLE</DefineConstants>
+ <DefineConstants>$(DefineConstants);FEATURE_SERIALIZABLE;FEATURE_CONCURRENTMERGESCHEDULER;FEATURE_THREAD_PRIORITY</DefineConstants>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.xproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.xproj b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.xproj
new file mode 100644
index 0000000..ac422c3
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.xproj
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+ <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+ </PropertyGroup>
+ <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" />
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>afbd39f9-4c9f-4cc2-bf2f-296d7993ca32</ProjectGuid>
+ <RootNamespace>Lucene.Net.Benchmarks</RootNamespace>
+ <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath>
+ <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+ <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+ </PropertyGroup>
+ <PropertyGroup>
+ <SchemaVersion>2.0</SchemaVersion>
+ </PropertyGroup>
+ <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
index 6d62a2f..0884588 100644
--- a/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
@@ -11,6 +11,7 @@
// OF ANY KIND, either express or implied; not even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+using Lucene.Net.Support;
using Sax.Helpers;
using System;
using System.Text;
@@ -143,7 +144,7 @@ namespace TagSoup
{
return "http://www.w3.org/XML/1998/namespace";
}
- return string.Intern("urn:x-prefix:" + prefix);
+ return "urn:x-prefix:" + prefix.Intern();
}
/// <summary>
@@ -158,7 +159,7 @@ namespace TagSoup
{
return name;
}
- return string.Intern(name.Substring(colon + 1));
+ return name.Substring(colon + 1).Intern();
}
/// <summary>
@@ -193,7 +194,7 @@ namespace TagSoup
int i = atts.GetIndex(name);
if (i == -1)
{
- name = string.Intern(name);
+ name = name.Intern();
if (type == null)
{
type = "CDATA";
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
index ff47d0d..45eebfe 100644
--- a/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
@@ -73,7 +73,7 @@ namespace TagSoup
public void EOF(char[] buff, int offset, int length)
{
- theWriter.Close();
+ theWriter.Dispose();
}
public void ETag(char[] buff, int offset, int length)
@@ -177,7 +177,7 @@ namespace TagSoup
public void EndDocument()
{
- theWriter.Close();
+ theWriter.Dispose();
}
public void EndElement(string uri, string localname, string qname)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
index a0a5463..e353b85 100644
--- a/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
@@ -542,7 +542,7 @@ namespace TagSoup
/// <returns></returns>
private Stream GetInputStream(string publicid, string systemid)
{
- var basis = new Uri("file://" + Environment.CurrentDirectory + Path.DirectorySeparatorChar);
+ var basis = new Uri("file://" + Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar);
var url = new Uri(basis, systemid);
return new FileStream(url.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read);
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Benchmark/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/project.json b/src/Lucene.Net.Benchmark/project.json
index adac6d5..95b896f 100644
--- a/src/Lucene.Net.Benchmark/project.json
+++ b/src/Lucene.Net.Benchmark/project.json
@@ -18,35 +18,36 @@
"nowarn": [ "1591", "1573" ]
},
"dependencies": {
- "icu.net": "54.1.1-alpha",
+ "icu.net": "54.1.1-alpha",
"Lucene.Net": "4.8.0",
"Lucene.Net.Analysis.Common": "4.8.0",
- "Lucene.Net.Facet": "4.8.0",
- "Lucene.Net.Highlighter": "4.8.0",
- "Lucene.Net.ICU": "4.8.0",
- "Lucene.Net.Queries": "4.8.0",
- "Lucene.Net.QueryParser": "4.8.0",
- "Lucene.Net.Spatial": "4.8.0",
- "Sax.Net": "2.0.2",
- "SharpZipLib": "0.86.0",
- "Spatial4n.Core": "0.4.1-beta00003",
- "TagSoup.Net": "1.2.1.1"
+ "Lucene.Net.Facet": "4.8.0",
+ "Lucene.Net.Highlighter": "4.8.0",
+ "Lucene.Net.ICU": "4.8.0",
+ "Lucene.Net.Queries": "4.8.0",
+ "Lucene.Net.QueryParser": "4.8.0",
+ "Lucene.Net.Spatial": "4.8.0",
+ "Spatial4n.Core": "0.4.1-beta00003"
},
"frameworks": {
"netstandard1.5": {
- "imports": "dnxcore50",
+ "imports": [ "dnxcore50", "portable-net403+sl5+win8+wp8+wpa81" ],
"buildOptions": {
"debugType": "portable",
"define": [ "NETSTANDARD" ]
},
"dependencies": {
- "NETStandard.Library": "1.6.0"
+ "NETStandard.Library": "1.6.0",
+ "SharpZipLib.NETStandard": "0.86.0.1"
}
},
"net451": {
"buildOptions": {
"debugType": "full",
"define": [ "FEATURE_SERIALIZABLE" ]
+ },
+ "dependencies": {
+ "SharpZipLib": "0.86.0"
}
}
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.ICU/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.ICU/project.json b/src/Lucene.Net.ICU/project.json
index f075fc7..9d718b6 100644
--- a/src/Lucene.Net.ICU/project.json
+++ b/src/Lucene.Net.ICU/project.json
@@ -24,6 +24,12 @@
"../Lucene.Net.Analysis.Common/Analysis/Th/ThaiWordFilterFactory.cs",
"../Lucene.Net.Analysis.Common/Analysis/Util/CharArrayIterator.cs",
"../Lucene.Net.Analysis.Common/Analysis/Util/SegmentingTokenizerBase.cs",
+ "../Lucene.Net.Analysis.ICU/Collation/ICUCollationAttributeFactory.cs",
+ "../Lucene.Net.Analysis.ICU/Collation/ICUCollationDocValuesField.cs",
+ "../Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyAnalyzer.cs",
+ "../Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilter.cs",
+ "../Lucene.Net.Analysis.ICU/Collation/ICUCollationKeyFilterFactory.cs",
+ "../Lucene.Net.Analysis.ICU/Collation/TokenAttributes/ICUCollatedTermAttributeImpl.cs",
"../Lucene.Net.Highlighter/PostingsHighlight/DefaultPassageFormatter.cs",
"../Lucene.Net.Highlighter/PostingsHighlight/MultiTermHighlighting.cs",
"../Lucene.Net.Highlighter/PostingsHighlight/Passage.cs",
@@ -53,7 +59,8 @@
"define": [ "NETSTANDARD" ]
},
"dependencies": {
- "NETStandard.Library": "1.6.0"
+ "NETStandard.Library": "1.6.0",
+ "System.Globalization": "4.3.0"
}
},
"net451": {
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs b/src/Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs
index 55b0b3b..1edc777 100644
--- a/src/Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs
+++ b/src/Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs
@@ -85,6 +85,9 @@ namespace Lucene.Net.Collation
[Test]
public void TestThreadSafe()
{
+#if NETSTANDARD
+ fail("LUCENENET TODO: Fatal crash on NETSTANDARD");
+#endif
int iters = 20 * RANDOM_MULTIPLIER;
for (int i = 0; i < iters; i++)
{
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
index 301c807..e919fb1 100644
--- a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
@@ -103,11 +103,11 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
props["directory"] = "RAMDirectory";
if (setNormsProp)
{
- props["doc.tokenized.norms"] = normsPropVal.ToString(CultureInfo.InvariantCulture);
+ props["doc.tokenized.norms"] = normsPropVal.ToString();
}
if (setBodyNormsProp)
{
- props["doc.body.tokenized.norms"] = bodyNormsVal.ToString(CultureInfo.InvariantCulture);
+ props["doc.body.tokenized.norms"] = bodyNormsVal.ToString();
}
// Create PerfRunData
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
index 220d75e..310a88c 100644
--- a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
@@ -131,7 +131,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
Dictionary<string, string> props = new Dictionary<string, string>();
props["print.props"] = "false";
- props["content.source.forever"] = forever.ToString(CultureInfo.InvariantCulture);
+ props["content.source.forever"] = forever.ToString();
Config config = new Config(props);
EnwikiContentSource source = new StringableEnwikiSource(docs);
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.xproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.xproj b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.xproj
new file mode 100644
index 0000000..4b6c842
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.xproj
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+ <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+ </PropertyGroup>
+ <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" />
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>5356412f-8fc5-41d3-83c3-807740b06748</ProjectGuid>
+ <RootNamespace>Lucene.Net.Benchmarks</RootNamespace>
+ <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath>
+ <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+ <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+ </PropertyGroup>
+ <PropertyGroup>
+ <SchemaVersion>2.0</SchemaVersion>
+ </PropertyGroup>
+ <ItemGroup>
+ <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+ </ItemGroup>
+ <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs b/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
index 2e12258..2c61f0c 100644
--- a/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
+++ b/src/Lucene.Net.Tests.Benchmark/Properties/AssemblyInfo.cs
@@ -8,10 +8,6 @@ using System.Runtime.InteropServices;
[assembly: AssemblyTitle("Lucene.Net.Tests.Benchmark")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
-[assembly: AssemblyCompany("")]
-[assembly: AssemblyProduct("Lucene.Net.Tests.Benchmark")]
-[assembly: AssemblyCopyright("Copyright © 2017")]
-[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
@@ -22,15 +18,4 @@ using System.Runtime.InteropServices;
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("9257f543-44e2-4db6-8b27-a8a354c13e5b")]
-// Version information for an assembly consists of the following four values:
-//
-// Major Version
-// Minor Version
-// Build Number
-// Revision
-//
-// You can specify all the values or you can default the Build and Revision Numbers
-// by using the '*' as shown below:
-// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.0.0")]
-[assembly: AssemblyFileVersion("1.0.0.0")]
+// NOTE: Version information is in CommonAssemblyInfo.cs
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.Benchmark/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/project.json b/src/Lucene.Net.Tests.Benchmark/project.json
index 5babcd7..e4c79ab 100644
--- a/src/Lucene.Net.Tests.Benchmark/project.json
+++ b/src/Lucene.Net.Tests.Benchmark/project.json
@@ -9,21 +9,21 @@
"includeFiles": [
"ByTask/Feeds/trecdocs.zip",
"ByTask/conf.zip",
- "ByTask/reuters.first20.lines.txt",
- "ByTask/test-mapping-ISOLatin1Accent-partial.txt",
+ "ByTask/reuters.first20.lines.txt",
+ "ByTask/test-mapping-ISOLatin1Accent-partial.txt",
"Quality/reuters.578.lines.txt.bz2",
- "Quality/trecQRels.txt",
- "Quality/trecTopics.txt"
+ "Quality/trecQRels.txt",
+ "Quality/trecTopics.txt"
]
}
},
"dependencies": {
"dotnet-test-nunit-teamcity": "3.4.0-beta-3",
- "icu.net": "54.1.1-alpha",
- "Lucene.Net.Analysis.Common": "4.8.0",
- "Lucene.Net.Benchmark": "4.8.0",
+ "icu.net": "54.1.1-alpha",
+ "Lucene.Net.Analysis.Common": "4.8.0",
+ "Lucene.Net.Benchmark": "4.8.0",
"Lucene.Net.Facet": "4.8.0",
- "Lucene.Net.Highlighter": "4.8.0",
+ "Lucene.Net.Highlighter": "4.8.0",
"Lucene.Net.ICU": "4.8.0",
"Lucene.Net.TestFramework": "4.8.0",
"NUnit": "3.5.0"
@@ -31,7 +31,7 @@
"testRunner": "nunit-teamcity",
"frameworks": {
"netcoreapp1.0": {
- "imports": "dnxcore50",
+ "imports": [ "dnxcore50", "portable-net403+sl5+win8+wp8+wpa81" ],
"buildOptions": {
"debugType": "portable",
"define": [ "NETSTANDARD" ]
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4cb35e92/src/Lucene.Net.Tests.ICU/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.ICU/project.json b/src/Lucene.Net.Tests.ICU/project.json
index 1c3c0cd..15471b0 100644
--- a/src/Lucene.Net.Tests.ICU/project.json
+++ b/src/Lucene.Net.Tests.ICU/project.json
@@ -11,6 +11,10 @@
"../Lucene.Net.Tests.Analysis.Common/Analysis/Th/TestThaiWordFilterFactory.cs",
"../Lucene.Net.Tests.Analysis.Common/Analysis/Util/TestCharArrayIterator.cs",
"../Lucene.Net.Tests.Analysis.Common/Analysis/Util/TestSegmentingTokenizerBase.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationDocValuesField.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyFilter.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyFilterFactory.cs",
"../Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs",
"../Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs",
"../Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs",
@@ -50,6 +54,10 @@
"../Lucene.Net.Tests.Analysis.Common/Analysis/Th/TestThaiWordFilterFactory.cs",
"../Lucene.Net.Tests.Analysis.Common/Analysis/Util/TestCharArrayIterator.cs",
"../Lucene.Net.Tests.Analysis.Common/Analysis/Util/TestSegmentingTokenizerBase.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationDocValuesField.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyAnalyzer.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyFilter.cs",
+ "../Lucene.Net.Tests.Analysis.ICU/Collation/TestICUCollationKeyFilterFactory.cs",
"../Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs",
"../Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs",
"../Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs",
[24/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
Lucene.Net.Benchmark: Added Sax and TagSoup to the Support folder.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/198e5868
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/198e5868
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/198e5868
Branch: refs/heads/master
Commit: 198e586874c849e98df78477abf56d1902151222
Parents: 6cf6a6e
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Aug 4 00:21:07 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 19:33:28 2017 +0700
----------------------------------------------------------------------
.../ByTask/Feeds/DemoHTMLParser.cs | 8 +-
.../ByTask/Feeds/EnwikiContentSource.cs | 9 +-
.../Lucene.Net.Benchmark.csproj | 60 +
.../Lucene.Net.Benchmark.project.json | 4 +-
.../Support/EnglishNumberFormatExtensions.cs | 48 +-
.../Support/Sax/Attributes.cs | 219 ++
.../Support/Sax/ContentHandler.cs | 364 +++
.../Support/Sax/DTDHandler.cs | 100 +
.../Support/Sax/EntityResolver.cs | 109 +
.../Support/Sax/ErrorHandler.cs | 122 +
.../Support/Sax/Ext/Attributes2.cs | 108 +
.../Support/Sax/Ext/Attributes2Impl.cs | 277 ++
.../Support/Sax/Ext/DeclHandler.cs | 131 +
.../Support/Sax/Ext/DefaultHandler2.cs | 112 +
.../Support/Sax/Ext/EntityResolver2.cs | 178 ++
.../Support/Sax/Ext/LexicalHandler.cs | 180 ++
.../Support/Sax/Ext/Locator2.cs | 64 +
.../Support/Sax/Ext/Locator2Impl.cs | 76 +
.../Support/Sax/Helpers/AttributesImpl.cs | 615 ++++
.../Support/Sax/Helpers/DefaultHandler.cs | 389 +++
.../Support/Sax/Helpers/LocatorImpl.cs | 131 +
.../Support/Sax/Helpers/NamespaceSupport.cs | 841 +++++
.../Support/Sax/Helpers/XMLFilterImpl.cs | 587 ++++
.../Support/Sax/InputSource.cs | 242 ++
src/Lucene.Net.Benchmark/Support/Sax/Locator.cs | 125 +
.../Support/Sax/SAXException.cs | 165 +
.../Support/Sax/SAXNotRecognizedException.cs | 66 +
.../Support/Sax/SAXNotSupportedException.cs | 67 +
.../Support/Sax/SAXParseException.cs | 269 ++
.../Support/Sax/XMLFilter.cs | 41 +
.../Support/Sax/XMLReader.cs | 305 ++
.../Support/StringExtensions.cs | 14 +
.../Support/TagSoup/AutoDetector.cs | 41 +
.../Support/TagSoup/Element.cs | 215 ++
.../Support/TagSoup/ElementType.cs | 269 ++
.../Support/TagSoup/HTMLScanner.cs | 745 +++++
.../Support/TagSoup/HTMLSchema.Generated.cs | 2910 ++++++++++++++++++
.../Support/TagSoup/HTMLSchema.tt | 72 +
.../Support/TagSoup/PYXScanner.cs | 138 +
.../Support/TagSoup/PYXWriter.cs | 286 ++
.../Support/TagSoup/Parser.cs | 1484 +++++++++
.../Support/TagSoup/ScanHandler.cs | 105 +
.../Support/TagSoup/Scanner.cs | 53 +
.../Support/TagSoup/Schema.cs | 159 +
.../Support/TagSoup/XMLReader.cs | 1567 ++++++++++
.../Support/TagSoup/definitions/html.stml | 249 ++
.../Support/TagSoup/definitions/html.tssl | 2762 +++++++++++++++++
.../Support/TagSoup/stml/stml.rnc | 49 +
.../Support/TagSoup/stml/stml.xslt | 150 +
.../Support/TagSoup/tssl/tssl-models.xslt | 47 +
.../Support/TagSoup/tssl/tssl-validate.xslt | 40 +
.../Support/TagSoup/tssl/tssl.rnc | 75 +
.../Support/TagSoup/tssl/tssl.xslt | 220 ++
53 files changed, 17627 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
index 0903754..2ee6184 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
@@ -1,8 +1,8 @@
// LUCENENET TODO: Use HTML Agility pack instead of SAX ?
using Lucene.Net.Support;
-using Sax.Net;
-using Sax.Net.Helpers;
+using Sax;
+using Sax.Helpers;
using System;
using System.Collections.Generic;
using System.IO;
@@ -64,9 +64,9 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
public Parser(InputSource source)
{
- TagSoup.Net.Parser parser = new TagSoup.Net.Parser();
+ TagSoup.Parser parser = new TagSoup.Parser();
- parser.SetFeature(TagSoup.Net.Parser.NAMESPACES_FEATURE, true);
+ parser.SetFeature(TagSoup.Parser.NAMESPACES_FEATURE, true);
StringBuilder title = new StringBuilder(), body = new StringBuilder();
DefaultHandler handler = new DefaultHandlerAnonymousHelper(this, title, body);
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
index 2cb24d0..6870b3b 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
@@ -3,8 +3,8 @@
using Lucene.Net.Benchmarks.ByTask.Utils;
using Lucene.Net.Support.Threading;
using Lucene.Net.Util;
-using Sax.Net;
-using Sax.Net.Helpers;
+using Sax;
+using Sax.Helpers;
using System;
using System.Collections.Generic;
using System.Globalization;
@@ -195,9 +195,10 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
try
{
- Sax.Net.IXmlReader reader = new TagSoup.Net.XmlReaderFactory().CreateXmlReader(); //XMLReaderFactory.createXMLReader();
+ Sax.IXMLReader reader = new TagSoup.Parser(); //XMLReaderFactory.createXMLReader();
reader.ContentHandler = this;
reader.ErrorHandler = this;
+
while (!stopped)
{
Stream localFileIS = outerInstance.@is;
@@ -293,7 +294,6 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
}
}
}
-
}
private static readonly IDictionary<string, int?> ELEMENTS = new Dictionary<string, int?>();
@@ -350,6 +350,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
parser.Stop();
if (@is != null)
{
+ Thread.Sleep(1); // LUCENENET: Allow parser to stop before Dispose() is called
@is.Dispose();
@is = null;
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
index f00cd18..2645226 100644
--- a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
@@ -160,6 +160,49 @@
<Compile Include="Quality\Utils\SimpleQQParser.cs" />
<Compile Include="Quality\Utils\SubmissionReport.cs" />
<Compile Include="Support\EnglishNumberFormatExtensions.cs" />
+ <Compile Include="Support\Sax\Attributes.cs" />
+ <Compile Include="Support\Sax\ContentHandler.cs" />
+ <Compile Include="Support\Sax\DTDHandler.cs" />
+ <Compile Include="Support\Sax\EntityResolver.cs" />
+ <Compile Include="Support\Sax\ErrorHandler.cs" />
+ <Compile Include="Support\Sax\Ext\Attributes2.cs" />
+ <Compile Include="Support\Sax\Ext\Attributes2Impl.cs" />
+ <Compile Include="Support\Sax\Ext\DeclHandler.cs" />
+ <Compile Include="Support\Sax\Ext\DefaultHandler2.cs" />
+ <Compile Include="Support\Sax\Ext\EntityResolver2.cs" />
+ <Compile Include="Support\Sax\Ext\Locator2.cs" />
+ <Compile Include="Support\Sax\Ext\LexicalHandler.cs" />
+ <Compile Include="Support\Sax\Ext\Locator2Impl.cs" />
+ <Compile Include="Support\Sax\Helpers\AttributesImpl.cs" />
+ <Compile Include="Support\Sax\Helpers\DefaultHandler.cs" />
+ <Compile Include="Support\Sax\Helpers\LocatorImpl.cs" />
+ <Compile Include="Support\Sax\Helpers\NamespaceSupport.cs" />
+ <Compile Include="Support\Sax\Helpers\XMLFilterImpl.cs" />
+ <Compile Include="Support\Sax\InputSource.cs" />
+ <Compile Include="Support\Sax\XMLReader.cs" />
+ <Compile Include="Support\Sax\Locator.cs" />
+ <Compile Include="Support\Sax\SAXException.cs" />
+ <Compile Include="Support\Sax\SAXNotRecognizedException.cs" />
+ <Compile Include="Support\Sax\SAXNotSupportedException.cs" />
+ <Compile Include="Support\Sax\SAXParseException.cs" />
+ <Compile Include="Support\Sax\XMLFilter.cs" />
+ <Compile Include="Support\StringExtensions.cs" />
+ <Compile Include="Support\TagSoup\AutoDetector.cs" />
+ <Compile Include="Support\TagSoup\Element.cs" />
+ <Compile Include="Support\TagSoup\ElementType.cs" />
+ <Compile Include="Support\TagSoup\HTMLScanner.cs" />
+ <Compile Include="Support\TagSoup\HTMLSchema.Generated.cs">
+ <AutoGen>True</AutoGen>
+ <DesignTime>True</DesignTime>
+ <DependentUpon>HTMLSchema.tt</DependentUpon>
+ </Compile>
+ <Compile Include="Support\TagSoup\Parser.cs" />
+ <Compile Include="Support\TagSoup\PYXScanner.cs" />
+ <Compile Include="Support\TagSoup\PYXWriter.cs" />
+ <Compile Include="Support\TagSoup\ScanHandler.cs" />
+ <Compile Include="Support\TagSoup\Scanner.cs" />
+ <Compile Include="Support\TagSoup\Schema.cs" />
+ <Compile Include="Support\TagSoup\XMLReader.cs" />
<Compile Include="Utils\ExtractReuters.cs" />
<Compile Include="Utils\ExtractWikipedia.cs" />
<Compile Include="..\CommonAssemblyInfo.cs">
@@ -202,8 +245,25 @@
</ItemGroup>
<ItemGroup>
<None Include="Lucene.Net.Benchmark.project.json" />
+ <None Include="Support\TagSoup\definitions\html.tssl" />
+ <None Include="Support\TagSoup\stml\stml.rnc" />
+ <None Include="Support\TagSoup\tssl\tssl.rnc" />
+ </ItemGroup>
+ <ItemGroup>
+ <Content Include="Support\TagSoup\definitions\html.stml" />
+ <Content Include="Support\TagSoup\HTMLSchema.tt">
+ <Generator>TextTemplatingFileGenerator</Generator>
+ <LastGenOutput>HTMLSchema.Generated.cs</LastGenOutput>
+ </Content>
+ <Content Include="Support\TagSoup\stml\stml.xslt" />
+ <Content Include="Support\TagSoup\tssl\tssl-models.xslt" />
+ <Content Include="Support\TagSoup\tssl\tssl-validate.xslt" />
+ <Content Include="Support\TagSoup\tssl\tssl.xslt" />
</ItemGroup>
<ItemGroup />
+ <ItemGroup>
+ <Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
+ </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
index 0a83392..f764f6a 100644
--- a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
@@ -4,10 +4,8 @@
},
"dependencies": {
"icu.net": "54.1.1-alpha",
- "Sax.Net": "2.0.2",
"SharpZipLib": "0.86.0",
- "Spatial4n.Core": "0.4.1-beta00003",
- "TagSoup.Net": "1.2.1.1"
+ "Spatial4n.Core": "0.4.1-beta00003"
},
"frameworks": {
"net451": {}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs b/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
index 71362f0..1d99a8b 100644
--- a/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
+++ b/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
@@ -10,12 +10,12 @@ namespace Lucene.Net.Support
/// </summary>
public static class EnglishNumberFormatExtensions
{
- private const long Quadrillion = Trillion * 1000;
- private const long Trillion = Billion * 1000;
- private const long Billion = Million * 1000;
- private const long Million = Thousand * 1000;
- private const long Thousand = Hundred * 10;
- private const long Hundred = 100;
+ private const long QUADRILLION = TRILLION * 1000;
+ private const long TRILLION = BILLION * 1000;
+ private const long BILLION = MILLION * 1000;
+ private const long MILLION = THOUSAND * 1000;
+ private const long THOUSAND = HUNDRED * 10;
+ private const long HUNDRED = 100;
/// <summary>
/// Returns the spelled-out English words for the provided <paramref name="value"/>.
@@ -44,60 +44,60 @@ namespace Lucene.Net.Support
long unit = 0;
- if (value >= Quadrillion)
+ if (value >= QUADRILLION)
{
- unit = (value / Quadrillion);
- value -= unit * Quadrillion;
+ unit = (value / QUADRILLION);
+ value -= unit * QUADRILLION;
ToWords(unit, builder);
builder.Append(" quadrillion");
if (value > 0) builder.Append(" ");
}
- if (value >= Trillion)
+ if (value >= TRILLION)
{
- unit = (value / Trillion);
- value -= unit * Trillion;
+ unit = (value / TRILLION);
+ value -= unit * TRILLION;
ToWords(unit, builder);
builder.Append(" trillion");
if (value > 0) builder.Append(" ");
}
- if (value >= Billion)
+ if (value >= BILLION)
{
- unit = (value / Billion);
- value -= unit * Billion;
+ unit = (value / BILLION);
+ value -= unit * BILLION;
ToWords(unit, builder);
builder.Append(" billion");
if (value > 0) builder.Append(" ");
}
- if (value >= Million)
+ if (value >= MILLION)
{
- unit = (value / Million);
- value -= unit * Million;
+ unit = (value / MILLION);
+ value -= unit * MILLION;
ToWords(unit, builder);
builder.Append(" million");
if (value > 0) builder.Append(" ");
}
- if (value >= Thousand)
+ if (value >= THOUSAND)
{
- unit = (value / Thousand);
- value -= unit * Thousand;
+ unit = (value / THOUSAND);
+ value -= unit * THOUSAND;
ToWords(unit, builder);
builder.Append(" thousand");
if (value > 0) builder.Append(" ");
}
- if (value >= Hundred)
+ if (value >= HUNDRED)
{
- unit = (value / Hundred);
- value -= unit * Hundred;
+ unit = (value / HUNDRED);
+ value -= unit * HUNDRED;
ToWords(unit, builder);
builder.Append(" hundred");
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Attributes.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Attributes.cs b/src/Lucene.Net.Benchmark/Support/Sax/Attributes.cs
new file mode 100644
index 0000000..5a51ae3
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Attributes.cs
@@ -0,0 +1,219 @@
+// Attributes.java - attribute list with Namespace support
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the public domain.
+// $Id: Attributes.java,v 1.13 2004/03/18 12:28:05 dmegginson Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Interface for a list of XML attributes.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This interface allows access to a list of attributes in
+ /// three different ways:
+ /// <list type="number">
+ /// <item><description>by attribute index;</description></item>
+ /// <item><description>by Namespace-qualified name; or</description></item>
+ /// <item><description>by qualified (prefixed) name.</description></item>
+ /// </list>
+ /// <para/>
+ /// The list will not contain attributes that were declared
+ /// #IMPLIED but not specified in the start tag. It will also not
+ /// contain attributes used as Namespace declarations(xmlns*) unless
+ /// the <a href="http://xml.org/sax/features/namespace-prefixes">http://xml.org/sax/features/namespace-prefixes</a>
+ /// feature is set to <var>true</var> (it is <var>false</var> by
+ /// default).
+ /// Because SAX2 conforms to the original "Namespaces in XML"
+ /// recommendation, it normally does not
+ /// give namespace declaration attributes a namespace URI.
+ /// <para/>
+ /// Some SAX2 parsers may support using an optional feature flag
+ /// (<a href="http://xml.org/sax/features/xmlns-uris">http://xml.org/sax/features/xmlns-uris</a>) to request
+ /// that those attributes be given URIs, conforming to a later
+ /// backwards-incompatible revision of that recommendation. (The
+ /// attribute's "local name" will be the prefix, or "xmlns" when
+ /// defining a default element namespace.) For portability, handler
+ /// code should always resolve that conflict, rather than requiring
+ /// parsers that can change the setting of that feature flag.
+ /// If the namespace-prefixes feature (see above) is
+ /// <var>false</var>, access by qualified name may not be available; if
+ /// the<code>http://xml.org/sax/features/namespaces</code> feature is
+ /// <var>false</var>, access by Namespace-qualified names may not be
+ /// available.
+ /// <para/>This interface replaces the now-deprecated SAX1 { @link
+ /// org.xml.sax.AttributeList AttributeList } interface, which does not
+ /// contain Namespace support.In addition to Namespace support, it
+ /// adds the<var> getIndex</var> methods (below).
+ /// <para/>The order of attributes in the list is unspecified, and will
+ /// vary from implementation to implementation.
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="Helpers.Attributes"/>
+ /// <seealso cref="Ext.IDeclHandler"/>
+ public interface IAttributes
+ {
+ ////////////////////////////////////////////////////////////////////
+ // Indexed access.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Return the number of attributes in the list.
+ /// <para/>
+ /// Once you know the number of attributes, you can iterate
+ /// through the list.
+ /// </summary>
+ /// <returns>The number of attributes in the list.</returns>
+ /// <seealso cref="GetURI(int)"/>
+ /// <seealso cref="GetLocalName(int)"/>
+ /// <seealso cref="GetQName(int)"/>
+ /// <seealso cref="GetType(int)"/>
+ /// <seealso cref="GetValue(int)"/>
+ int Length { get; }
+
+ /// <summary>
+ /// Look up an attribute's Namespace URI by index.
+ /// </summary>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>The Namespace URI, or the empty string if none
+ /// is available, or null if the index is out of
+ /// range.</returns>
+ /// <seealso cref="GetURI(int)"/>
+ string GetURI(int index);
+
+ /// <summary>
+ /// Look up an attribute's local name by index.
+ /// </summary>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>The local name, or the empty string if Namespace
+ /// processing is not being performed, or null
+ /// if the index is out of range.</returns>
+ /// <seealso cref="Length"/>
+ string GetLocalName(int index);
+
+ /// <summary>
+ /// Look up an attribute's XML qualified (prefixed) name by index.
+ /// </summary>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>The XML qualified name, or the empty string
+ /// if none is available, or null if the index
+ /// is out of range.</returns>
+ /// <seealso cref="Length"/>
+ string GetQName(int index);
+
+ /// <summary>
+ /// Look up an attribute's type by index.
+ /// </summary>
+ /// <remarks>
+ /// The attribute type is one of the strings "CDATA", "ID",
+ /// "IDREF", "IDREFS", "NMTOKEN", "NMTOKENS", "ENTITY", "ENTITIES",
+ /// or "NOTATION" (always in upper case).
+ /// <para/>
+ /// If the parser has not read a declaration for the attribute,
+ /// or if the parser does not report attribute types, then it must
+ /// return the value "CDATA" as stated in the XML 1.0 Recommendation
+ /// (clause 3.3.3, "Attribute-Value Normalization").
+ /// <para/>
+ /// For an enumerated attribute that is not a notation, the
+ /// parser will report the type as "NMTOKEN".
+ /// </remarks>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>The attribute's type as a string, or null if the
+ /// index is out of range.</returns>
+ /// <seealso cref="Length"/>
+ string GetType(int index);
+
+ /// <summary>
+ /// Look up an attribute's value by index.
+ /// </summary>
+ /// <remarks>
+ /// If the attribute value is a list of tokens (IDREFS,
+ /// ENTITIES, or NMTOKENS), the tokens will be concatenated
+ /// into a single string with each token separated by a
+ /// single space.
+ /// </remarks>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>The attribute's value as a string, or null if the
+ /// index is out of range.</returns>
+ /// <seealso cref="Length"/>
+ string GetValue(int index);
+
+ ////////////////////////////////////////////////////////////////////
+ // Name-based query.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Look up the index of an attribute by Namespace name.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty string if
+ /// the name has no Namespace URI.</param>
+ /// <param name="localName">The attribute's local name.</param>
+ /// <returns>The index of the attribute, or -1 if it does not
+ /// appear in the list.</returns>
+ int GetIndex(string uri, string localName);
+
+ /// <summary>
+ /// Look up the index of an attribute by XML qualified (prefixed) name.
+ /// </summary>
+ /// <param name="qName">The qualified (prefixed) name.</param>
+ /// <returns>The index of the attribute, or -1 if it does not
+ /// appear in the list.</returns>
+ int GetIndex(string qName);
+
+ /// <summary>
+ /// Look up an attribute's type by Namespace name.
+ /// <para/>
+ /// See <see cref="GetType(int)"/> for a description
+ /// of the possible types.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty String if the
+ /// name has no Namespace URI.</param>
+ /// <param name="localName">The local name of the attribute.</param>
+ /// <returns>The attribute type as a string, or null if the
+ /// attribute is not in the list or if Namespace
+ /// processing is not being performed.</returns>
+ string GetType(string uri, string localName);
+
+ /// <summary>
+ /// Look up an attribute's type by XML qualified (prefixed) name.
+ /// <para/>
+ /// See <see cref="GetType(int)"/> for a description
+ /// of the possible types.
+ /// </summary>
+ /// <param name="qName">The XML qualified name.</param>
+ /// <returns>The attribute type as a string, or null if the
+ /// attribute is not in the list or if qualified names
+ /// are not available.</returns>
+ string GetType(string qName);
+
+ /// <summary>
+ /// Look up an attribute's value by Namespace name.
+ /// <para/>
+ /// See <see cref="GetValue(int)"/> for a description
+ /// of the possible values.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty String if the
+ /// name has no Namespace URI.</param>
+ /// <param name="localName">The local name of the attribute.</param>
+ /// <returns>The attribute value as a string, or null if the
+ /// attribute is not in the list.</returns>
+ string GetValue(string uri, string localName);
+
+ /// <summary>
+ /// Look up an attribute's value by XML qualified (prefixed) name.
+ /// <para/>
+ /// See <see cref="GetValue(int)"/> for a description
+ /// of the possible values.
+ /// </summary>
+ /// <param name="qName"></param>
+ /// <returns></returns>
+ string GetValue(string qName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/ContentHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/ContentHandler.cs b/src/Lucene.Net.Benchmark/Support/Sax/ContentHandler.cs
new file mode 100644
index 0000000..3f03aae
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/ContentHandler.cs
@@ -0,0 +1,364 @@
+// ContentHandler.java - handle main document content.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the public domain.
+// $Id: ContentHandler.java,v 1.13 2004/04/26 17:50:49 dmegginson Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Receive notification of the logical content of a document.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This is the main interface that most SAX applications
+ /// implement: if the application needs to be informed of basic parsing
+ /// events, it implements this interface and registers an instance with
+ /// the SAX parser using the <see cref="IXMLReader.ContentHandler"/>
+ /// method.The parser uses the instance to report
+ /// basic document-related events like the start and end of elements
+ /// and character data.
+ /// <para/>The order of events in this interface is very important, and
+ /// mirrors the order of information in the document itself.For
+ /// example, all of an element's content (character data, processing
+ /// instructions, and/or subelements) will appear, in order, between
+ /// the startElement event and the corresponding endElement event.</p>
+ /// <p>This interface is similar to the now-deprecated SAX 1.0
+ /// DocumentHandler interface, but it adds support for Namespaces
+ /// and for reporting skipped entities(in non-validating XML
+ /// processors).
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1+ (sax2r3pre1)</version>
+ /// <seealso cref="IXMLReader"/>
+ /// <seealso cref="IDTDHandler"/>
+ /// <seealso cref="IErrorHandler"/>
+ public interface IContentHandler
+ {
+ /// <summary>
+ /// Receive an object for locating the origin of SAX document events.
+ /// </summary>
+ /// <remarks>
+ /// SAX parsers are strongly encouraged (though not absolutely
+ /// required) to supply a locator: if it does so, it must supply
+ /// the locator to the application by invoking this method before
+ /// invoking any of the other methods in the ContentHandler
+ /// interface.
+ /// <para/>
+ /// The locator allows the application to determine the end
+ /// position of any document-related event, even if the parser is
+ /// not reporting an error. Typically, the application will
+ /// use this information for reporting its own errors (such as
+ /// character content that does not match an application's
+ /// business rules). The information returned by the locator
+ /// is probably not sufficient for use with a search engine.
+ /// <para/>
+ /// Note that the locator will return correct information only
+ /// during the invocation SAX event callbacks after
+ /// <see cref="StartDocument()"/> returns and before
+ /// <see cref="EndDocument()"/> is called. The
+ /// application should not attempt to use it at any other time.
+ /// </remarks>
+ /// <param name="locator">an object that can return the location of
+ /// any SAX document event</param>
+ /// <seealso cref="ILocator"/>
+ void SetDocumentLocator(ILocator locator);
+
+ /// <summary>
+ /// Receive notification of the beginning of a document.
+ /// </summary>
+ /// <remarks>
+ /// The SAX parser will invoke this method only once, before any
+ /// other event callbacks (except for <see cref="SetDocumentLocator(ILocator)"/>.
+ /// </remarks>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ /// <seealso cref="EndDocument()"/>
+ void StartDocument();
+
+ /// <summary>
+ /// Receive notification of the end of a document.
+ /// </summary>
+ /// <remarks>
+ /// <strong>There is an apparent contradiction between the
+ /// documentation for this method and the documentation for
+ /// <see cref="IErrorHandler.FatalError(SAXParseException)"/>. Until this ambiguity is
+ /// resolved in a future major release, clients should make no
+ /// assumptions about whether <see cref="EndDocument()"/> will or will not be
+ /// invoked when the parser has reported a <see cref="IErrorHandler.FatalError(SAXParseException)"/> or thrown
+ /// an exception.</strong>
+ /// <para/>The SAX parser will invoke this method only once, and it will
+ /// be the last method invoked during the parse.The parser shall
+ /// not invoke this method until it has either abandoned parsing
+ /// (because of an unrecoverable error) or reached the end of
+ /// input.
+ /// </remarks>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ /// <seealso cref="StartDocument()"/>
+ void EndDocument();
+
+ /// <summary>
+ /// Begin the scope of a prefix-URI Namespace mapping.
+ /// </summary>
+ /// <remarks>
+ /// The information from this event is not necessary for
+ /// normal Namespace processing: the SAX XML reader will
+ /// automatically replace prefixes for element and attribute
+ /// names when the<code>http://xml.org/sax/features/namespaces</code>
+ /// feature is <var>true</var> (the default).
+ /// <para/>
+ /// There are cases, however, when applications need to
+ /// use prefixes in character data or in attribute values,
+ /// where they cannot safely be expanded automatically; the
+ /// start/endPrefixMapping event supplies the information
+ /// to the application to expand prefixes in those contexts
+ /// itself, if necessary.
+ /// <para/>Note that start/endPrefixMapping events are not
+ /// guaranteed to be properly nested relative to each other:
+ /// all startPrefixMapping events will occur immediately before the
+ /// corresponding <see cref="StartElement(string, string, string, IAttributes)"/> event,
+ /// and all <see cref="EndPrefixMapping(string)"/>
+ /// events will occur immediately after the corresponding
+ /// <see cref="EndElement(string, string, string)"/> event,
+ /// but their order is not otherwise
+ /// guaranteed.
+ /// <para/>There should never be start/endPrefixMapping events for the
+ /// "xml" prefix, since it is predeclared and immutable.
+ /// </remarks>
+ /// <param name="prefix">the Namespace prefix being declared.
+ /// An empty string is used for the default element namespace,
+ /// which has no prefix.</param>
+ /// <param name="uri">the Namespace URI the prefix is mapped to</param>
+ /// <exception cref="SAXException">the client may throw
+ /// an exception during processing</exception>
+ /// <seealso cref="EndPrefixMapping(string)"/>
+ /// <seealso cref="StartElement(string, string, string, IAttributes)"/>
+ void StartPrefixMapping(string prefix, string uri);
+
+ /// <summary>
+ /// End the scope of a prefix-URI mapping.
+ /// </summary>
+ /// <remarks>
+ /// See <see cref="StartPrefixMapping(string, string)"/> for
+ /// details. These events will always occur immediately after the
+ /// corresponding <see cref="EndElement(string, string, string)"/> event, but the order of
+ /// <see cref="EndPrefixMapping(string)"/> events is not otherwise
+ /// guaranteed.
+ /// </remarks>
+ /// <param name="prefix">the prefix that was being mapped.
+ /// This is the empty string when a default mapping scope ends.</param>
+ /// <exception cref="SAXException">the client may throw
+ /// an exception during processing</exception>
+ /// <seealso cref="EndPrefixMapping(string)"/>
+ /// <seealso cref="EndElement(string, string, string)"/>
+ void EndPrefixMapping(string prefix);
+
+ /// <summary>
+ /// Receive notification of the beginning of an element.
+ /// <para/>
+ /// The Parser will invoke this method at the beginning of every
+ /// element in the XML document; there will be a corresponding
+ /// <see cref="EndElement(string, string, string)"/> event for every <see cref="StartElement(string, string, string, IAttributes)"/> event
+ /// (even when the element is empty). All of the element's content will be
+ /// reported, in order, before the corresponding <see cref="EndElement(string, string, string)"/>
+ /// event.
+ /// <para/>
+ /// This event allows up to three name components for each element:
+ /// <list type="number">
+ /// <item><description>the Namespace URI;</description></item>
+ /// <item><description>the local name; and</description></item>
+ /// <item><description>the qualified (prefixed) name.</description></item>
+ /// </list>
+ /// <para/>
+ /// Any or all of these may be provided, depending on the
+ /// values of the<var> http://xml.org/sax/features/namespaces</var>
+ /// and the<var>http://xml.org/sax/features/namespace-prefixes</var>
+ /// properties:
+ /// <list type="bullet">
+ /// <item><description>the Namespace URI and local name are required when
+ /// the namespaces property is <var>true</var> (the default), and are
+ /// optional when the namespaces property is <var>false</var> (if one is
+ /// specified, both must be);</description></item>
+ /// <item><description>the qualified name is required when the namespace-prefixes property
+ /// is <var>true</var>, and is optional when the namespace-prefixes property
+ /// is <var>false</var> (the default).</description></item>
+ /// </list>
+ /// <para/>Note that the attribute list provided will contain only
+ /// attributes with explicit values (specified or defaulted):
+ /// #IMPLIED attributes will be omitted. The attribute list
+ /// will contain attributes used for Namespace declarations
+ /// (xmlns/// attributes) only if the
+ /// <a href="http://xml.org/sax/features/namespace-prefixes">http://xml.org/sax/features/namespace-prefixes</a>
+ /// property is true (it is false by default, and support for a
+ /// true value is optional).
+ /// <para/>Like <see cref="Characters(char[], int, int)"/>, attribute values may have
+ /// characters that need more than one <c>char</c> value.
+ /// </summary>
+ /// <param name="uri">uri the Namespace URI, or the empty string if the
+ /// element has no Namespace URI or if Namespace
+ /// processing is not being performed</param>
+ /// <param name="localName">the local name (without prefix), or the
+ /// empty string if Namespace processing is not being
+ /// performed</param>
+ /// <param name="qName">the qualified name (with prefix), or the
+ /// empty string if qualified names are not available</param>
+ /// <param name="atts">the attributes attached to the element. If
+ /// there are no attributes, it shall be an empty
+ /// <see cref="IAttributes"/> object. The value of this object after
+ /// <see cref="StartElement(string, string, string, IAttributes)"/> returns is undefined</param>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ /// <seealso cref="EndElement(string, string, string)"/>
+ /// <seealso cref="IAttributes"/>
+ /// <seealso cref="Helpers.Attributes"/>
+ void StartElement(string uri, string localName, string qName, IAttributes atts);
+
+ /// <summary>
+ /// Receive notification of the end of an element.
+ /// </summary>
+ /// <remarks>
+ /// The SAX parser will invoke this method at the end of every
+ /// element in the XML document; there will be a corresponding
+ /// <see cref="StartElement(string, string, string, IAttributes)"/> event for every endElement
+ /// event (even when the element is empty).
+ /// <para/>
+ /// For information on the names, see <see cref="StartElement(string, string, string, IAttributes)"/>.
+ /// </remarks>
+ /// <param name="uri">the Namespace URI, or the empty string if the
+ /// element has no Namespace URI or if Namespace
+ /// processing is not being performed</param>
+ /// <param name="localName">the local name (without prefix), or the
+ /// empty string if Namespace processing is not being
+ /// performed</param>
+ /// <param name="qName">the qualified XML name (with prefix), or the
+ /// empty string if qualified names are not available</param>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ void EndElement(string uri, string localName, string qName);
+
+ /// <summary>
+ /// Receive notification of character data.
+ /// </summary>
+ /// <remarks>
+ /// The Parser will call this method to report each chunk of
+ /// character data. SAX parsers may return all contiguous character
+ /// data in a single chunk, or they may split it into several
+ /// chunks; however, all of the characters in any single event
+ /// must come from the same external entity so that the Locator
+ /// provides useful information.
+ /// <para/>
+ /// The application must not attempt to read from the array
+ /// outside of the specified range.
+ /// <para/>
+ /// Individual characters may consist of more than one Java
+ /// <c>char</c> value.There are two important cases where this
+ /// happens, because characters can't be represented in just sixteen bits.
+ /// In one case, characters are represented in a <em>Surrogate Pair</em>,
+ /// using two special Unicode values. Such characters are in the so-called
+ /// "Astral Planes", with a code point above U+FFFF.A second case involves
+ /// composite characters, such as a base character combining with one or
+ /// more accent characters.
+ /// <para/> Your code should not assume that algorithms using
+ /// <c>char</c>-at-a-time idioms will be working in character
+ /// units; in some cases they will split characters. This is relevant
+ /// wherever XML permits arbitrary characters, such as attribute values,
+ /// processing instruction data, and comments as well as in data reported
+ /// from this method. It's also generally relevant whenever Java code
+ /// manipulates internationalized text; the issue isn't unique to XML.
+ /// <para/>Note that some parsers will report whitespace in element
+ /// content using the <see cref="IgnorableWhitespace(char[], int, int)"/>
+ /// method rather than this one (validating parsers <em>must</em>
+ /// do so).
+ /// </remarks>
+ /// <param name="ch">the characters from the XML document</param>
+ /// <param name="start">the start position in the array</param>
+ /// <param name="length">the number of characters to read from the array</param>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ /// <seealso cref="IgnorableWhitespace(char[], int, int)"/>
+ /// <seealso cref="ILocator"/>
+ void Characters(char[] ch, int start, int length);
+
+ /// <summary>
+ /// Receive notification of ignorable whitespace in element content.
+ /// </summary>
+ /// <remarks>
+ /// Validating Parsers must use this method to report each chunk
+ /// of whitespace in element content (see the W3C XML 1.0
+ /// recommendation, section 2.10): non-validating parsers may also
+ /// use this method if they are capable of parsing and using
+ /// content models.
+ /// <para/>
+ /// SAX parsers may return all contiguous whitespace in a single
+ /// chunk, or they may split it into several chunks; however, all of
+ /// the characters in any single event must come from the same
+ /// external entity, so that the Locator provides useful
+ /// information.
+ /// <para/>
+ /// The application must not attempt to read from the array
+ /// outside of the specified range.
+ /// </remarks>
+ /// <param name="ch">the characters from the XML document</param>
+ /// <param name="start">the start position in the array</param>
+ /// <param name="length">the number of characters to read from the array</param>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ /// <seealso cref="Characters(char[], int, int)"/>
+ void IgnorableWhitespace(char[] ch, int start, int length);
+
+ /// <summary>
+ /// Receive notification of a processing instruction.
+ /// </summary>
+ /// <remarks>
+ /// The Parser will invoke this method once for each processing
+ /// instruction found: note that processing instructions may occur
+ /// before or after the main document element.
+ /// <para/>
+ /// A SAX parser must never report an XML declaration(XML 1.0,
+ /// section 2.8) or a text declaration(XML 1.0, section 4.3.1)
+ /// using this method.
+ /// <para/>
+ /// Like <see cref="Characters(char[], int, int)"/>, processing instruction
+ /// data may have characters that need more than one <c>char</c>
+ /// value.
+ /// </remarks>
+ /// <param name="target">the processing instruction target</param>
+ /// <param name="data">the processing instruction data, or null if
+ /// none was supplied. The data does not include any
+ /// whitespace separating it from the target</param>
+ /// <exception cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</exception>
+ void ProcessingInstruction(string target, string data);
+
+ /// <summary>
+ /// Receive notification of a skipped entity.
+ /// This is not called for entity references within markup constructs
+ /// such as element start tags or markup declarations. (The XML
+ /// recommendation requires reporting skipped external entities.
+ /// SAX also reports internal entity expansion/non-expansion, except
+ /// within markup constructs.)
+ /// <para/>
+ /// The Parser will invoke this method each time the entity is
+ /// skipped.Non-validating processors may skip entities if they
+ /// have not seen the declarations(because, for example, the
+ /// entity was declared in an external DTD subset). All processors
+ /// may skip external entities, depending on the values of the
+ /// <a href="http://xml.org/sax/features/external-general-entities">http://xml.org/sax/features/external-general-entities</a>
+ /// and the <a href="http://xml.org/sax/features/external-general-entities">http://xml.org/sax/features/external-general-entities</a>
+ /// properties.
+ /// </summary>
+ /// <param name="name">the name of the skipped entity. If it is a
+ /// parameter entity, the name will begin with '%', and if
+ /// it is the external DTD subset, it will be the string
+ /// "[dtd]"</param>
+ /// <seealso cref="SAXException">any SAX exception, possibly
+ /// wrapping another exception</seealso>
+ void SkippedEntity(string name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/DTDHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/DTDHandler.cs b/src/Lucene.Net.Benchmark/Support/Sax/DTDHandler.cs
new file mode 100644
index 0000000..b96777b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/DTDHandler.cs
@@ -0,0 +1,100 @@
+// SAX DTD handler.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: DTDHandler.java,v 1.8 2002/01/30 21:13:43 dbrownell Exp $
+
+
+namespace Sax
+{
+ /// <summary>
+ /// Receive notification of basic DTD-related events.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// See <a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// If a SAX application needs information about notations and
+ /// unparsed entities, then the application implements this
+ /// interface and registers an instance with the SAX parser using
+ /// the parser's setDTDHandler method. The parser uses the
+ /// instance to report notation and unparsed entity declarations to
+ /// the application.
+ /// <para/>
+ /// Note that this interface includes only those DTD events that
+ /// the XML recommendation<em>requires</em> processors to report:
+ /// notation and unparsed entity declarations.
+ /// <para/>
+ /// The SAX parser may report these events in any order, regardless
+ /// of the order in which the notations and unparsed entities were
+ /// declared; however, all DTD events must be reported after the
+ /// document handler's startDocument event, and before the first
+ /// startElement event.
+ /// (If the <see cref="Ext.ILexicalHandler"/> is
+ /// used, these events must also be reported before the endDTD event.)
+ /// <para/>
+ /// It is up to the application to store the information for
+ /// future use(perhaps in a hash table or object tree).
+ /// If the application encounters attributes of type "NOTATION",
+ /// "ENTITY", or "ENTITIES", it can use the information that it
+ /// obtained through this interface to find the entity and/or
+ /// notation corresponding with the attribute value.
+ /// </remarks>
+ /// <seealso cref="IXMLReader.SetDTDHandler"/>
+ public interface IDTDHandler
+ {
+ /// <summary>
+ /// Receive notification of a notation declaration event.
+ /// </summary>
+ /// <remarks>
+ /// It is up to the application to record the notation for later
+ /// reference, if necessary;
+ /// notations may appear as attribute values and in unparsed entity
+ /// declarations, and are sometime used with processing instruction
+ /// target names.
+ /// <para/>
+ /// At least one of publicId and systemId must be non-null.
+ /// If a system identifier is present, and it is a URL, the SAX
+ /// parser must resolve it fully before passing it to the
+ /// application through this event.
+ /// <para/>
+ /// There is no guarantee that the notation declaration will be
+ /// reported before any unparsed entities that use it.
+ /// </remarks>
+ /// <param name="name">The notation name.</param>
+ /// <param name="publicId">The notation's public identifier, or <c>null</c> if none was given.</param>
+ /// <param name="systemId">The notation's system identifier, or <c>null</c> if none was given.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping another exception.</exception>
+ /// <seealso cref="UnparsedEntityDecl(string, string, string, string)"/>
+ /// <seealso cref="IAttributes"/>
+ void NotationDecl(string name,
+ string publicId,
+ string systemId);
+
+ /// <summary>
+ /// Receive notification of an unparsed entity declaration event.
+ /// </summary>
+ /// <remarks>
+ /// Note that the notation name corresponds to a notation
+ /// reported by the <see cref="NotationDecl(string, string, string)"/> event.
+ /// It is up to the application to record the entity for later
+ /// reference, if necessary;
+ /// unparsed entities may appear as attribute values.
+ /// <para/>
+ /// If the system identifier is a URL, the parser must resolve it
+ /// fully before passing it to the application.
+ /// </remarks>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping another exception.</exception>
+ /// <param name="name">The unparsed entity's name.</param>
+ /// <param name="publicId">The entity's public identifier, or null if none was given.</param>
+ /// <param name="systemId">The entity's system identifier.</param>
+ /// <param name="notationName">The name of the associated notation.</param>
+ /// <seealso cref="NotationDecl(string, string, string)"/>
+ /// <seealso cref="IAttributes"/>
+ void UnparsedEntityDecl(string name,
+ string publicId,
+ string systemId,
+ string notationName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/EntityResolver.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/EntityResolver.cs b/src/Lucene.Net.Benchmark/Support/Sax/EntityResolver.cs
new file mode 100644
index 0000000..a66bedd
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/EntityResolver.cs
@@ -0,0 +1,109 @@
+// SAX entity resolver.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: EntityResolver.java,v 1.10 2002/01/30 21:13:44 dbrownell Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Basic interface for resolving entities.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// If a SAX application needs to implement customized handling
+ /// for external entities, it must implement this interface and
+ /// register an instance with the SAX driver using the
+ /// <see cref="IXMLReader.EntityResolver"/>
+ /// property.
+ /// <para/>
+ /// The XML reader will then allow the application to intercept any
+ /// external entities(including the external DTD subset and external
+ /// parameter entities, if any) before including them.
+ /// <para/>
+ /// Many SAX applications will not need to implement this interface,
+ /// but it will be especially useful for applications that build
+ /// XML documents from databases or other specialised input sources,
+ /// or for applications that use URI types other than URLs.
+ /// <para/>
+ /// The following resolver would provide the application
+ /// with a special character stream for the entity with the system
+ /// identifier "http://www.myhost.com/today":
+ ///
+ /// <code>
+ /// public class MyResolver : IEntityResolver
+ /// {
+ /// public InputSource ResolveEntity (string publicId, string systemId)
+ /// {
+ /// if (systemId.Equals("http://www.myhost.com/today", StringComparison.Ordinal))
+ /// {
+ /// // return a special input source
+ /// MyReader reader = new MyReader();
+ /// return new InputSource(reader);
+ /// }
+ /// else
+ /// {
+ /// // use the default behaviour
+ /// return null;
+ /// }
+ /// }
+ /// }
+ /// </code>
+ /// <para/>
+ /// The application can also use this interface to redirect system
+ /// identifiers to local URIs or to look up replacements in a catalog
+ /// (possibly by using the public identifier).
+ /// </remarks>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <since>SAX 1.0</since>
+ /// <seealso cref="IXMLReader.EntityResolver"/>
+ /// <seealso cref="InputSource"/>
+ public interface IEntityResolver
+ {
+ /// <summary>
+ /// Allow the application to resolve external entities.
+ /// </summary>
+ /// <remarks>
+ /// The parser will call this method before opening any external
+ /// entity except the top-level document entity.Such entities include
+ /// the external DTD subset and external parameter entities referenced
+ /// within the DTD(in either case, only if the parser reads external
+ /// parameter entities), and external general entities referenced
+ /// within the document element(if the parser reads external general
+ /// entities). The application may request that the parser locate
+ /// the entity itself, that it use an alternative URI, or that it
+ /// use data provided by the application(as a character or byte
+ /// input stream).
+ /// <para/>
+ /// Application writers can use this method to redirect external
+ /// system identifiers to secure and/or local URIs, to look up
+ /// public identifiers in a catalogue, or to read an entity from a
+ /// database or other input source(including, for example, a dialog
+ /// box). Neither XML nor SAX specifies a preferred policy for using
+ /// public or system IDs to resolve resources.However, SAX specifies
+ /// how to interpret any InputSource returned by this method, and that
+ /// if none is returned, then the system ID will be dereferenced as
+ /// a URL.
+ /// <para/>
+ /// If the system identifier is a URL, the SAX parser must
+ /// resolve it fully before reporting it to the application.
+ /// </remarks>
+ /// <param name="publicId">The public identifier of the external entity
+ /// being referenced, or null if none was supplied.</param>
+ /// <param name="systemId">The system identifier of the external entity
+ /// being referenced.</param>
+ /// <returns>An InputSource object describing the new input source,
+ /// or null to request that the parser open a regular
+ /// URI connection to the system identifier.</returns>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping another exception.</exception>
+ /// <exception cref="IOException">A .NET-specific IO exception, possibly the result of creating
+ /// a new <see cref="InputStream"/> or <see cref="System.IO.TextReader"/> for the <see cref="InputSource"/>.</exception>
+ /// <seealso cref="InputSource"/>
+ InputSource ResolveEntity(string publicId,
+ string systemId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/ErrorHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/ErrorHandler.cs b/src/Lucene.Net.Benchmark/Support/Sax/ErrorHandler.cs
new file mode 100644
index 0000000..785e697
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/ErrorHandler.cs
@@ -0,0 +1,122 @@
+// SAX error handler.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: ErrorHandler.java,v 1.10 2004/03/08 13:01:00 dmegginson Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Basic interface for SAX error handlers.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// If a SAX application needs to implement customized error
+ /// handling, it must implement this interface and then register an
+ /// instance with the XML reader using the
+ /// <see cref="IXMLReader.ErrorHandler"/>
+ /// property. The parser will then report all errors and warnings
+ /// through this interface.
+ /// <para/>
+ /// <strong>WARNING:</strong> If an application does <em>not</em>
+ /// register an ErrorHandler, XML parsing errors will go unreported,
+ /// except that<em> SAXParseException</em>s will be thrown for fatal errors.
+ /// In order to detect validity errors, an ErrorHandler that does something
+ /// with <see cref="Error(SAXParseException)"/> calls must be registered.
+ /// <para/>
+ /// For XML processing errors, a SAX driver must use this interface
+ /// in preference to throwing an exception: it is up to the application
+ /// to decide whether to throw an exception for different types of
+ /// errors and warnings.Note, however, that there is no requirement that
+ /// the parser continue to report additional errors after a call to
+ /// <see cref="FatalError(SAXParseException)"/>. In other words, a SAX driver class
+ /// may throw an exception after reporting any fatalError.
+ /// Also parsers may throw appropriate exceptions for non - XML errors.
+ /// For example, <see cref="IXMLReader.Parse()"/> would throw
+ /// an <see cref="System.IO.IOException"/> for errors accessing entities or the document.
+ /// </remarks>
+ /// <since>SAX 1.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1+ (sax2r3pre1)</version>
+ /// <seealso cref="IXMLReader.ErrorHandler"/>
+ /// <seealso cref="SAXParseException"/>
+ public interface IErrorHandler
+ {
+ /// <summary>
+ /// Receive notification of a warning.
+ /// </summary>
+ /// <remarks>
+ /// SAX parsers will use this method to report conditions that
+ /// are not errors or fatal errors as defined by the XML
+ /// recommendation.The default behaviour is to take no
+ /// action.
+ /// <para/>
+ /// The SAX parser must continue to provide normal parsing events
+ /// after invoking this method: it should still be possible for the
+ /// application to process the document through to the end.
+ /// <para/>
+ /// Filters may use this method to report other, non-XML warnings
+ /// as well.
+ /// </remarks>
+ /// <param name="exception">The warning information encapsulated in a SAX parse exception.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping another exception.</exception>
+ /// <seealso cref="SAXParseException"/>
+ void Warning(SAXParseException exception);
+
+ /// <summary>
+ /// Receive notification of a recoverable error.
+ /// </summary>
+ /// <remarks>
+ /// This corresponds to the definition of "error" in section 1.2
+ /// of the W3C XML 1.0 Recommendation.For example, a validating
+ /// parser would use this callback to report the violation of a
+ /// validity constraint.The default behaviour is to take no
+ /// action.
+ /// <para/>
+ /// The SAX parser must continue to provide normal parsing
+ /// events after invoking this method: it should still be possible
+ /// for the application to process the document through to the end.
+ /// If the application cannot do so, then the parser should report
+ /// a fatal error even if the XML recommendation does not require
+ /// it to do so.
+ /// <para/>
+ /// Filters may use this method to report other, non-XML errors
+ /// as well.
+ /// </remarks>
+ /// <param name="exception">The error information encapsulated in a SAX parse exception.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping another exception.</exception>
+ /// <seealso cref="SAXParseException"/>
+ void Error(SAXParseException exception);
+
+ /// <summary>
+ /// Receive notification of a non-recoverable error.
+ /// </summary>
+ /// <remarks>
+ /// <strong>There is an apparent contradiction between the
+ /// documentation for this method and the documentation for
+ /// <see cref="IContentHandler.EndDocument()"/>. Until this ambiguity
+ /// is resolved in a future major release, clients should make no
+ /// assumptions about whether EndDocument() will or will not be
+ /// invoked when the parser has reported a FatalError() or thrown
+ /// an exception.</strong>
+ /// <para/>
+ /// This corresponds to the definition of "fatal error" in
+ /// section 1.2 of the W3C XML 1.0 Recommendation.For example, a
+ /// parser would use this callback to report the violation of a
+ /// well-formedness constraint.
+ /// <para/>
+ /// The application must assume that the document is unusable
+ /// after the parser has invoked this method, and should continue
+ /// (if at all) only for the sake of collecting additional error
+ /// messages: in fact, SAX parsers are free to stop reporting any
+ /// other events once this method has been invoked.
+ /// </remarks>
+ /// <param name="exception">The error information encapsulated in a SAX parse exception.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping another exception.</exception>
+ /// <seealso cref="SAXParseException"/>
+ void FatalError(SAXParseException exception);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2.cs
new file mode 100644
index 0000000..87248dd
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2.cs
@@ -0,0 +1,108 @@
+// Attributes2.java - extended Attributes
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: Attributes2.java,v 1.6 2004/03/08 13:01:00 dmegginson Exp $
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// SAX2 extension to augment the per-attribute information
+ /// provided though <see cref="IAttributes"/>.
+ /// If an implementation supports this extension, the attributes
+ /// provided in <see cref="IContentHandler"/>
+ /// will implement this interface,
+ /// and the<em> http://xml.org/sax/features/use-attributes2</em>
+ /// feature flag will have the value<em>true</em>.
+ /// <para/>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// <para/>
+ /// XMLReader implementations are not required to support this
+ /// information, and it is not part of core-only SAX2 distributions.
+ /// <para/>
+ /// Note that if an attribute was defaulted (<em>!IsSpecified()</em>)
+ /// it will of necessity also have been declared(<em>IsDeclared()</em>)
+ /// in the DTD.
+ /// Similarly if an attribute's type is anything except CDATA, then it
+ /// must have been declared.
+ /// </summary>
+ /// <since>SAX 2.0 (extensions 1.1 alpha)</since>
+ /// <author>David Brownell</author>
+ /// <version>TBS</version>
+ public interface IAttributes2 : IAttributes
+ {
+ /// <summary>
+ /// Returns false unless the attribute was declared in the DTD.
+ /// This helps distinguish two kinds of attributes that SAX reports
+ /// as CDATA: ones that were declared (and hence are usually valid),
+ /// and those that were not (and which are never valid).
+ /// </summary>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>true if the attribute was declared in the DTD, false otherwise.</returns>
+ /// <exception cref="IndexOutOfRangeException">When the supplied index does not identify an attribute.</exception>
+ bool IsDeclared(int index);
+
+ /// <summary>
+ /// Returns false unless the attribute was declared in the DTD.
+ /// This helps distinguish two kinds of attributes that SAX reports
+ /// as CDATA: ones that were declared (and hence are usually valid),
+ /// and those that were not (and which are never valid).
+ /// </summary>
+ /// <param name="qName">The XML qualified (prefixed) name.</param>
+ /// <returns>true if the attribute was declared in the DTD, false otherwise.</returns>
+ /// <exception cref="ArgumentException">When the supplied name does not identify an attribute.</exception>
+ bool IsDeclared(string qName);
+
+ /// <summary>
+ /// Returns false unless the attribute was declared in the DTD.
+ /// This helps distinguish two kinds of attributes that SAX reports
+ /// as CDATA: ones that were declared (and hence are usually valid),
+ /// and those that were not (and which are never valid).
+ /// </summary>
+ /// <remarks>
+ /// Remember that since DTDs do not "understand" namespaces, the
+ /// namespace URI associated with an attribute may not have come from
+ /// the DTD.The declaration will have applied to the attribute's
+ /// <em>qName</em>
+ /// </remarks>
+ /// <param name="uri">The Namespace URI, or the empty string if the name has no Namespace URI.</param>
+ /// <param name="localName">The attribute's local name.</param>
+ /// <returns>true if the attribute was declared in the DTD, false otherwise.</returns>
+ /// <exception cref="ArgumentException">When the supplied names do not identify an attribute.</exception>
+ bool IsDeclared(string uri, string localName);
+
+ /// <summary>
+ /// Returns true unless the attribute value was provided
+ /// by DTD defaulting.
+ /// </summary>
+ /// <param name="index">The attribute index (zero-based).</param>
+ /// <returns>true if the value was found in the XML text, false if the value was provided by DTD defaulting.</returns>
+ /// <exception cref="IndexOutOfRangeException">When the supplied index does not identify an attribute.</exception>
+ bool IsSpecified(int index);
+
+ /// <summary>
+ /// Returns true unless the attribute value was provided
+ /// by DTD defaulting.
+ /// </summary>
+ /// <remarks>
+ /// Remember that since DTDs do not "understand" namespaces, the
+ /// namespace URI associated with an attribute may not have come from
+ /// the DTD.The declaration will have applied to the attribute's
+ /// <em>qName</em>.
+ /// </remarks>
+ /// <param name="uri">The Namespace URI, or the empty string if the name has no Namespace URI.</param>
+ /// <param name="localName">The attribute's local name.</param>
+ /// <returns>true if the value was found in the XML text, false if the value was provided by DTD defaulting.</returns>
+ /// <exception cref="ArgumentException">When the supplied names do not identify an attribute.</exception>
+ bool IsSpecified(string uri, string localName);
+
+ /// <summary>
+ /// Returns true unless the attribute value was provided
+ /// by DTD defaulting.
+ /// </summary>
+ /// <param name="qName">The XML qualified (prefixed) name.</param>
+ /// <returns>true if the value was found in the XML text, false if the value was provided by DTD defaulting.</returns>
+ /// <exception cref="ArgumentException">When the supplied name does not identify an attribute.</exception>
+ bool IsSpecified(string qName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2Impl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2Impl.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2Impl.cs
new file mode 100644
index 0000000..b6c30ed
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Attributes2Impl.cs
@@ -0,0 +1,277 @@
+// Attributes2Impl.java - extended AttributesImpl
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: Attributes2Impl.java,v 1.5 2004/03/08 13:01:01 dmegginson Exp $
+
+using Sax.Helpers;
+using System;
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// SAX2 extension helper for additional Attributes information,
+ /// implementing the <see cref="Attributes2"/> interface.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// <para/>
+ /// This is not part of core-only SAX2 distributions.
+ /// <para/>
+ /// The <em>specified</em> flag for each attribute will always
+ /// be true, unless it has been set to false in the copy constructor
+ /// or using <see cref="SetSpecified(int, bool)"/>.
+ /// Similarly, the <em>declared</em> flag for each attribute will
+ /// always be false, except for defaulted attributes (<em>specified</em>
+ /// is false), non-CDATA attributes, or when it is set to true using
+ /// <see cref="SetDeclared(int, bool)"/>.
+ /// If you change an attribute's type by hand, you may need to modify
+ /// its <em>declared</em> flag to match.
+ /// </remarks>
+ /// <since>SAX 2.0 (extensions 1.1 alpha)</since>
+ /// <author>David Brownell</author>
+ /// <version>TBS</version>
+ public class Attributes2 : Attributes, IAttributes2
+ {
+ private bool[] declared;
+ private bool[] specified;
+
+
+ /// <summary>
+ /// Construct a new, empty <see cref="Attributes2"/> object.
+ /// </summary>
+ public Attributes2() { }
+
+
+ /// <summary>
+ /// Copy an existing Attributes or Attributes2 object.
+ /// If the object implements Attributes2, values of the
+ /// <em>specified</em> and <em>declared</em> flags for each
+ /// attribute are copied.
+ /// Otherwise the flag values are defaulted to assume no DTD was used,
+ /// unless there is evidence to the contrary (such as attributes with
+ /// type other than CDATA, which must have been <em>declared</em>).
+ /// <p>This constructor is especially useful inside a
+ /// <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/> event.</p>
+ /// </summary>
+ /// <param name="atts">The existing <see cref="IAttributes"/> object.</param>
+ public Attributes2(IAttributes atts)
+ : base(atts)
+ {
+ }
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of Attributes2
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Returns the current value of the attribute's "declared" flag.
+ /// </summary>
+ // javadoc mostly from interface
+ public bool IsDeclared(int index)
+ {
+ if (index < 0 || index >= Length)
+ throw new IndexOutOfRangeException(
+ "No attribute at index: " + index);
+ return declared[index];
+ }
+
+
+ /// <summary>
+ /// Returns the current value of the attribute's "declared" flag.
+ /// </summary>
+ // javadoc mostly from interface
+ public bool IsDeclared(string uri, string localName)
+ {
+ int index = GetIndex(uri, localName);
+
+ if (index < 0)
+ throw new ArgumentException(
+ "No such attribute: local=" + localName
+ + ", namespace=" + uri);
+ return declared[index];
+ }
+
+ /// <summary>
+ /// Returns the current value of the attribute's "declared" flag.
+ /// </summary>
+ // javadoc mostly from interface
+ public bool IsDeclared(string qName)
+ {
+ int index = GetIndex(qName);
+
+ if (index < 0)
+ throw new ArgumentException(
+ "No such attribute: " + qName);
+ return declared[index];
+ }
+
+ /// <summary>
+ /// Returns the current value of an attribute's "specified" flag.
+ /// </summary>
+ /// <param name="qName">The attribute index (zero-based).</param>
+ /// <returns>current flag value</returns>
+ /// <exception cref="IndexOutOfRangeException">When the supplied index does not identify an attribute.</exception>
+ public bool IsSpecified(int index)
+ {
+ if (index < 0 || index >= Length)
+ throw new IndexOutOfRangeException(
+ "No attribute at index: " + index);
+ return specified[index];
+ }
+
+ /// <summary>
+ /// Returns the current value of an attribute's "specified" flag.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty string if the name has no Namespace URI.</param>
+ /// <param name="localName">The attribute's local name.</param>
+ /// <returns>current flag value</returns>
+ /// <exception cref="ArgumentException">When the supplied names do not identify an attribute.</exception>
+ public bool IsSpecified(string uri, string localName)
+ {
+ int index = GetIndex(uri, localName);
+
+ if (index < 0)
+ throw new ArgumentException(
+ "No such attribute: local=" + localName
+ + ", namespace=" + uri);
+ return specified[index];
+ }
+
+ /// <summary>
+ /// Returns the current value of an attribute's "specified" flag.
+ /// </summary>
+ /// <param name="qName">The XML qualified (prefixed) name.</param>
+ /// <returns>current flag value</returns>
+ /// <exception cref="ArgumentException">When the supplied name does not identify an attribute.</exception>
+ public bool IsSpecified(string qName)
+ {
+ int index = GetIndex(qName);
+
+ if (index < 0)
+ throw new ArgumentException(
+ "No such attribute: " + qName);
+ return specified[index];
+ }
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Manipulators
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Copy an entire Attributes object. The "specified" flags are
+ /// assigned as true, and "declared" flags as false (except when
+ /// an attribute's type is not CDATA),
+ /// unless the object is an Attributes2 object.
+ /// In that case those flag values are all copied.
+ /// </summary>
+ /// <seealso cref="Attributes.SetAttributes(IAttributes)"/>
+ public override void SetAttributes(IAttributes atts)
+ {
+ int length = atts.Length;
+
+ base.SetAttributes(atts);
+ declared = new bool[length];
+ specified = new bool[length];
+
+ if (atts is Attributes2) {
+ Attributes2 a2 = (Attributes2)atts;
+ for (int i = 0; i < length; i++)
+ {
+ declared[i] = a2.IsDeclared(i);
+ specified[i] = a2.IsSpecified(i);
+ }
+ } else {
+ for (int i = 0; i < length; i++)
+ {
+ declared[i] = !"CDATA".Equals(atts.GetType(i));
+ specified[i] = true;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Add an attribute to the end of the list, setting its
+ /// "specified" flag to true. To set that flag's value
+ /// to false, use <see cref="SetSpecified(int, bool)"/>.
+ /// <p>Unless the attribute <em>type</em> is CDATA, this attribute
+ /// is marked as being declared in the DTD. To set that flag's value
+ /// to true for CDATA attributes, use <see cref="SetDeclared(int, bool)"/>.
+ /// </summary>
+ /// <seealso cref="Attributes.AddAttribute(string, string, string, string, string)"/>
+ public override void AddAttribute(string uri, string localName, string qName,
+ string type, string value)
+ {
+ base.AddAttribute(uri, localName, qName, type, value);
+
+ int length = Length;
+
+ if (length < specified.Length)
+ {
+ bool[] newFlags;
+
+ newFlags = new bool[length];
+ System.Array.Copy(declared, 0, newFlags, 0, declared.Length);
+ declared = newFlags;
+
+ newFlags = new bool[length];
+ System.Array.Copy(specified, 0, newFlags, 0, specified.Length);
+ specified = newFlags;
+ }
+
+ specified[length - 1] = true;
+ declared[length - 1] = !"CDATA".Equals(type);
+ }
+
+ // javadoc entirely from superclass
+ public override void RemoveAttribute(int index)
+ {
+ int origMax = Length - 1;
+
+ base.RemoveAttribute(index);
+ if (index != origMax)
+ {
+ System.Array.Copy(declared, index + 1, declared, index,
+ origMax - index);
+ System.Array.Copy(specified, index + 1, specified, index,
+ origMax - index);
+ }
+ }
+
+ /// <summary>
+ /// Assign a value to the "declared" flag of a specific attribute.
+ /// This is normally needed only for attributes of type CDATA,
+ /// including attributes whose type is changed to or from CDATA.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="value">The desired flag value.</param>
+ /// <exception cref="IndexOutOfRangeException">When the supplied index does not identify an attribute.</exception>
+ public virtual void SetDeclared(int index, bool value)
+ {
+ if (index < 0 || index >= Length)
+ throw new IndexOutOfRangeException(
+ "No attribute at index: " + index);
+ declared[index] = value;
+ }
+
+ /// <summary>
+ /// Assign a value to the "specified" flag of a specific attribute.
+ /// This is the only way this flag can be cleared, except clearing
+ /// by initialization with the copy constructor.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="value">The desired flag value.</param>
+ /// <exception cref="IndexOutOfRangeException">When the supplied index does not identify an attribute.</exception>
+ public virtual void SetSpecified(int index, bool value)
+ {
+ if (index < 0 || index >= Length)
+ throw new IndexOutOfRangeException(
+ "No attribute at index: " + index);
+ specified[index] = value;
+ }
+ }
+}
[18/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/XMLReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/XMLReader.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/XMLReader.cs
new file mode 100644
index 0000000..443348d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/XMLReader.cs
@@ -0,0 +1,1567 @@
+// XMLWriter.java - serialize an XML document.
+// Written by David Megginson, david@megginson.com
+// and placed by him into the public domain.
+// Extensively modified by John Cowan for TagSoup.
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+
+using Sax;
+using Sax.Ext;
+using Sax.Helpers;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// Filter to write an XML document from a SAX event stream.
+ /// </summary>
+ /// <remarks>
+ /// This class can be used by itself or as part of a SAX event
+ /// stream: it takes as input a series of SAX2 ContentHandler
+ /// events and uses the information in those events to write
+ /// an XML document. Since this class is a filter, it can also
+ /// pass the events on down a filter chain for further processing
+ /// (you can use the XMLWriter to take a snapshot of the current
+ /// state at any point in a filter chain), and it can be
+ /// used directly as a ContentHandler for a SAX2 XMLReader.
+ /// <para>
+ /// The client creates a document by invoking the methods for
+ /// standard SAX2 events, always beginning with the
+ /// <see cref="StartDocument()" /> method and ending with
+ /// the <see cref="EndDocument()" /> method. There are convenience
+ /// methods provided so that clients to not have to create empty
+ /// attribute lists or provide empty strings as parameters; for
+ /// example, the method invocation
+ /// </para>
+ /// <code>
+ /// w.StartElement("foo");
+ /// </code>
+ /// <para>is equivalent to the regular SAX2 ContentHandler method</para>
+ /// <code>
+ /// w.StartElement("", "foo", "", new Attributes());
+ /// </code>
+ /// <para>
+ /// Except that it is more efficient because it does not allocate
+ /// a new empty attribute list each time. The following code will send
+ /// a simple XML document to standard output:
+ /// </para>
+ /// <code>
+ /// XMLWriter w = new XMLWriter();
+ /// w.StartDocument();
+ /// w.StartElement("greeting");
+ /// w.Characters("Hello, world!");
+ /// w.EndElement("greeting");
+ /// w.EndDocument();
+ /// </code>
+ /// <para>The resulting document will look like this:</para>
+ /// <code>
+ /// <?xml version="1.0" standalone="yes"?>
+ /// <greeting>Hello, world!</greeting>
+ /// </code>
+ /// <para>
+ /// In fact, there is an even simpler convenience method,
+ /// <see cref="DataElement(string, string)"/>, designed for writing elements that
+ /// contain only character data, so the code to generate the
+ /// document could be shortened to
+ /// </para>
+ /// <code>
+ /// XMLWriter w = new XMLWriter();
+ /// w.StartDocument();
+ /// w.DataElement("greeting", "Hello, world!");
+ /// w.EndDocument();
+ /// </code>
+ /// <h2>Whitespace</h2>
+ /// <para>
+ /// According to the XML Recommendation, <em>all</em> whitespace
+ /// in an XML document is potentially significant to an application,
+ /// so this class never adds newlines or indentation. If you
+ /// insert three elements in a row, as in
+ /// </para>
+ /// <code>
+ /// w.DataElement("item", "1");
+ /// w.DataElement("item", "2");
+ /// w.DataElement("item", "3");
+ /// </code>
+ /// <para>you will end up with</para>
+ /// <code>
+ /// <item>1</item><item>3</item><item>3</item>
+ /// </code>
+ /// <para>
+ /// You need to invoke one of the <c>Characters</c> methods
+ /// explicitly to add newlines or indentation. Alternatively, you
+ /// can use <see cref="com.megginson.sax.DataWriter DataWriter" />, which
+ /// is derived from this class -- it is optimized for writing
+ /// purely data-oriented (or field-oriented) XML, and does automatic
+ /// linebreaks and indentation (but does not support mixed content
+ /// properly).
+ /// </para>
+ /// <h2>Namespace Support</h2>
+ /// <para>
+ /// The writer contains extensive support for XML Namespaces, so that
+ /// a client application does not have to keep track of prefixes and
+ /// supply <c>xmlns</c> attributes. By default, the XML writer will
+ /// generate Namespace declarations in the form _NS1, _NS2, etc., wherever
+ /// they are needed, as in the following example:
+ /// </para>
+ /// <code>
+ /// w.StartDocument();
+ /// w.EmptyElement("http://www.foo.com/ns/", "foo");
+ /// w.EndDocument();
+ /// </code>
+ /// <para>The resulting document will look like this:</para>
+ /// <code>
+ /// <?xml version="1.0" standalone="yes"?>
+ /// <_NS1:foo xmlns:_NS1="http://www.foo.com/ns/"/>
+ /// </code>
+ /// <para>
+ /// In many cases, document authors will prefer to choose their
+ /// own prefixes rather than using the (ugly) default names. The
+ /// XML writer allows two methods for selecting prefixes:
+ /// </para>
+ /// <list type="number">
+ /// <item><description>the qualified name</description></item>
+ /// <item><description>the <see cref="Prefix" /> property.</description></item>
+ /// </list>
+ /// <para>
+ /// Whenever the XML writer finds a new Namespace URI, it checks
+ /// to see if a qualified (prefixed) name is also available; if so
+ /// it attempts to use the name's prefix (as long as the prefix is
+ /// not already in use for another Namespace URI).
+ /// </para>
+ /// <para>
+ /// Before writing a document, the client can also pre-map a prefix
+ /// to a Namespace URI with the setPrefix method:
+ /// </para>
+ /// <code>
+ /// w.SetPrefix("http://www.foo.com/ns/", "foo");
+ /// w.StartDocument();
+ /// w.EmptyElement("http://www.foo.com/ns/", "foo");
+ /// w.EndDocument();
+ /// </code>
+ /// <para>The resulting document will look like this:</para>
+ /// <code>
+ /// <?xml version="1.0" standalone="yes"?>
+ /// <foo:foo xmlns:foo="http://www.foo.com/ns/"/>
+ /// </code>
+ /// <para>The default Namespace simply uses an empty string as the prefix:</para>
+ /// <code>
+ /// w.SetPrefix("http://www.foo.com/ns/", "");
+ /// w.StartDocument();
+ /// w.EmptyElement("http://www.foo.com/ns/", "foo");
+ /// w.EndDocument();
+ /// </code>
+ /// <para>The resulting document will look like this:</para>
+ /// <code>
+ /// <?xml version="1.0" standalone="yes"?>
+ /// <foo xmlns="http://www.foo.com/ns/"/>
+ /// </code>
+ /// <para>
+ /// By default, the XML writer will not declare a Namespace until
+ /// it is actually used. Sometimes, this approach will create
+ /// a large number of Namespace declarations, as in the following
+ /// example:
+ /// </para>
+ /// <code>
+ /// <xml version="1.0" standalone="yes"?>
+ /// <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ /// <rdf:Description about="http://www.foo.com/ids/books/12345">
+ /// <dc:title xmlns:dc="http://www.purl.org/dc/">A Dark Night</dc:title>
+ /// <dc:creator xmlns:dc="http://www.purl.org/dc/">Jane Smith</dc:title>
+ /// <dc:date xmlns:dc="http://www.purl.org/dc/">2000-09-09</dc:title>
+ /// </rdf:Description>
+ /// </rdf:RDF>
+ /// </code>
+ /// <para>
+ /// The "rdf" prefix is declared only once, because the RDF Namespace
+ /// is used by the root element and can be inherited by all of its
+ /// descendants; the "dc" prefix, on the other hand, is declared three
+ /// times, because no higher element uses the Namespace. To solve this
+ /// problem, you can instruct the XML writer to predeclare Namespaces
+ /// on the root element even if they are not used there:
+ /// </para>
+ /// <code>
+ /// w.ForceNSDecl("http://www.purl.org/dc/");
+ /// </code>
+ /// <para>
+ /// Now, the "dc" prefix will be declared on the root element even
+ /// though it's not needed there, and can be inherited by its
+ /// descendants:
+ /// </para>
+ /// <code>
+ /// <xml version="1.0" standalone="yes"?>
+ /// <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ /// xmlns:dc="http://www.purl.org/dc/">
+ /// <rdf:Description about="http://www.foo.com/ids/books/12345">
+ /// <dc:title>A Dark Night</dc:title>
+ /// <dc:creator>Jane Smith</dc:title>
+ /// <dc:date>2000-09-09</dc:title>
+ /// </rdf:Description>
+ /// </rdf:RDF>
+ /// </code>
+ /// <para>
+ /// This approach is also useful for declaring Namespace prefixes
+ /// that be used by qualified names appearing in attribute values or
+ /// character data.
+ /// </para>
+ /// </remarks>
+ /// <author>David Megginson, david@megginson.com</author>
+ /// <version>.2</version>
+ /// <seealso cref="IXMLFilter" />
+ /// <seealso cref="IContentHandler" />
+ public class XMLWriter : XMLFilter, ILexicalHandler
+ {
+ ////////////////////////////////////////////////////////////////////
+ // Constructors.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Create a new XML writer.
+ /// <para>Write to standard output.</para>
+ /// </summary>
+ public XMLWriter()
+ {
+ Init(null);
+ }
+
+ /// <summary>
+ /// Create a new XML writer.
+ /// <para>Write to the writer provided.</para>
+ /// </summary>
+ /// <param name="writer">
+ /// The output destination, or null to use standard
+ /// output.
+ /// </param>
+ public XMLWriter(TextWriter writer)
+ {
+ Init(writer);
+ }
+
+ /// <summary>
+ /// Create a new XML writer.
+ /// <para>Use the specified XML reader as the parent.</para>
+ /// </summary>
+ /// <param name="xmlreader">
+ /// The parent in the filter chain, or null
+ /// for no parent.
+ /// </param>
+ public XMLWriter(IXMLReader xmlreader) : base(xmlreader)
+ {
+ Init(null);
+ }
+
+ /// <summary>
+ /// Create a new XML writer.
+ /// <para>
+ /// Use the specified XML reader as the parent, and write
+ /// to the specified writer.
+ /// </para>
+ /// </summary>
+ /// <param name="xmlreader">
+ /// The parent in the filter chain, or null
+ /// for no parent.
+ /// </param>
+ /// <param name="writer">
+ /// The output destination, or null to use standard
+ /// output.
+ /// </param>
+ public XMLWriter(IXMLReader xmlreader, TextWriter writer) : base(xmlreader)
+ {
+ Init(writer);
+ }
+
+ public virtual void EndCDATA()
+ {
+ }
+
+ public virtual void EndDTD()
+ {
+ }
+
+ public virtual void EndEntity(string name)
+ {
+ }
+
+ public virtual void StartCDATA()
+ {
+ }
+
+ public virtual void StartDTD(string name, string publicid, string systemid)
+ {
+ if (name == null)
+ {
+ return; // can't cope
+ }
+ if (hasOutputDTD)
+ {
+ return; // only one DTD
+ }
+ hasOutputDTD = true;
+ Write("<!DOCTYPE ");
+ Write(name);
+ if (systemid == null)
+ {
+ systemid = "";
+ }
+ if (overrideSystem != null)
+ {
+ systemid = overrideSystem;
+ }
+ char sysquote = (systemid.IndexOf('"') != -1) ? '\'' : '"';
+ if (overridePublic != null)
+ {
+ publicid = overridePublic;
+ }
+ if (!(publicid == null || "".Equals(publicid)))
+ {
+ char pubquote = (publicid.IndexOf('"') != -1) ? '\'' : '"';
+ Write(" PUBLIC ");
+ Write(pubquote);
+ Write(publicid);
+ Write(pubquote);
+ Write(' ');
+ }
+ else
+ {
+ Write(" SYSTEM ");
+ }
+ Write(sysquote);
+ Write(systemid);
+ Write(sysquote);
+ Write(">\n");
+ }
+
+ public virtual void StartEntity(string name)
+ {
+ }
+
+ /// <summary>
+ /// Internal initialization method.
+ /// <para>All of the public constructors invoke this method.</para>
+ /// </summary>
+ /// <param name="writer">
+ /// The output destination, or null to use
+ /// standard output.
+ /// </param>
+ private void Init(TextWriter writer)
+ {
+ SetOutput(writer);
+ nsSupport = new NamespaceSupport();
+ prefixTable = new Hashtable();
+ forcedDeclTable = new Hashtable();
+ doneDeclTable = new Hashtable();
+ outputProperties = new Dictionary<string, string>();
+ }
+
+ /// <summary>
+ /// Reset the writer.
+ /// <para>
+ /// This method is especially useful if the writer throws an
+ /// exception before it is finished, and you want to reuse the
+ /// writer for a new document. It is usually a good idea to
+ /// invoke <see cref="Flush" /> before resetting the writer,
+ /// to make sure that no output is lost.
+ /// </para>
+ /// <para>
+ /// This method is invoked automatically by the
+ /// <see cref="StartDocument" /> method before writing
+ /// a new document.
+ /// </para>
+ /// <para>
+ /// <strong>Note:</strong> this method will <em>not</em>
+ /// clear the prefix or URI information in the writer or
+ /// the selected output writer.
+ /// </para>
+ /// </summary>
+ /// <seealso cref="Flush" />
+ public virtual void Reset()
+ {
+ elementLevel = 0;
+ prefixCounter = 0;
+ nsSupport.Reset();
+ }
+
+ /// <summary>
+ /// Flush the output.
+ /// <para>
+ /// This method flushes the output stream. It is especially useful
+ /// when you need to make certain that the entire document has
+ /// been written to output but do not want to close the output
+ /// stream.
+ /// </para>
+ /// <para>
+ /// This method is invoked automatically by the
+ /// <see cref="EndDocument" /> method after writing a
+ /// document.
+ /// </para>
+ /// </summary>
+ /// <seealso cref="Reset" />
+ public virtual void Flush()
+ {
+ output.Flush();
+ }
+
+ /// <summary>
+ /// Set a new output destination for the document.
+ /// </summary>
+ /// <param name="writer">
+ /// The output destination, or null to use
+ /// standard output.
+ /// </param>
+ /// <seealso cref="Flush" />
+ public virtual void SetOutput(TextWriter writer)
+ {
+ if (writer == null)
+ {
+ output = new StreamWriter(Console.OpenStandardOutput());
+ }
+ else
+ {
+ output = writer;
+ }
+ }
+
+ /// <summary>
+ /// Specify a preferred prefix for a Namespace URI.
+ /// <para>
+ /// Note that this method does not actually force the Namespace
+ /// to be declared; to do that, use the <see cref="ForceNSDecl(string)" />
+ /// method as well.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI.
+ /// </param>
+ /// <param name="prefix">
+ /// The preferred prefix, or "" to select
+ /// the default Namespace.
+ /// </param>
+ /// <seealso cref="GetPrefix" />
+ /// <seealso cref="ForceNSDecl(string)" />
+ /// <seealso cref="ForceNSDecl(string,string)" />
+ public virtual void SetPrefix(string uri, string prefix)
+ {
+ prefixTable[uri] = prefix;
+ }
+
+ /// <summary>
+ /// Get the current or preferred prefix for a Namespace URI.
+ /// </summary>
+ /// <param name="uri">The Namespace URI.</param>
+ /// <returns>The preferred prefix, or "" for the default Namespace.</returns>
+ /// <seealso cref="SetPrefix" />
+ public virtual string GetPrefix(string uri)
+ {
+ return (string)(prefixTable.ContainsKey(uri) ? prefixTable[uri] : string.Empty);
+ }
+
+ /// <summary>
+ /// Force a Namespace to be declared on the root element.
+ /// <para>
+ /// By default, the XMLWriter will declare only the Namespaces
+ /// needed for an element; as a result, a Namespace may be
+ /// declared many places in a document if it is not used on the
+ /// root element.
+ /// </para>
+ /// <para>
+ /// This method forces a Namespace to be declared on the root
+ /// element even if it is not used there, and reduces the number
+ /// of xmlns attributes in the document.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI to declare.
+ /// </param>
+ /// <seealso cref="ForceNSDecl(string,string)" />
+ /// <seealso cref="SetPrefix" />
+ public virtual void ForceNSDecl(string uri)
+ {
+ forcedDeclTable[uri] = true;
+ }
+
+ /// <summary>
+ /// Force a Namespace declaration with a preferred prefix.
+ /// <para>
+ /// This is a convenience method that invokes <see cref="SetPrefix" />
+ /// then <see cref="ForceNSDecl(string)" />.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI to declare on the root element.
+ /// </param>
+ /// <param name="prefix">
+ /// The preferred prefix for the Namespace, or ""
+ /// for the default Namespace.
+ /// </param>
+ /// <seealso cref="SetPrefix" />
+ /// <seealso cref="ForceNSDecl(string)" />
+ public virtual void ForceNSDecl(string uri, string prefix)
+ {
+ SetPrefix(uri, prefix);
+ ForceNSDecl(uri);
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Methods from Sax.5IContentHandler.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Write the XML declaration at the beginning of the document.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the XML declaration, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.StartDocument" />
+ public override void StartDocument()
+ {
+ Reset();
+ if (!("yes".Equals(outputProperties[OMIT_XML_DECLARATION] ?? "no")))
+ {
+ Write("<?xml");
+ if (version == null)
+ {
+ Write(" version=\"1.0\"");
+ }
+ else
+ {
+ Write(" version=\"");
+ Write(version);
+ Write("\"");
+ }
+ if (false == string.IsNullOrEmpty(outputEncoding))
+ {
+ Write(" encoding=\"");
+ Write(outputEncoding);
+ Write("\"");
+ }
+ if (standalone == null)
+ {
+ Write(" standalone=\"yes\"?>\n");
+ }
+ else
+ {
+ Write(" standalone=\"");
+ Write(standalone);
+ Write("\"");
+ }
+ }
+ base.StartDocument();
+ }
+
+ /// <summary>
+ /// Write a newline at the end of the document.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the newline, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.EndDocument" />
+ public override void EndDocument()
+ {
+ Write('\n');
+ base.EndDocument();
+ try
+ {
+ Flush();
+ }
+ catch (IOException e)
+ {
+ throw new SAXException(e.Message, e);
+ }
+ }
+
+ /// <summary>
+ /// Write a start tag.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI, or the empty string if none
+ /// is available.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local (unprefixed) name (required).
+ /// </param>
+ /// <param name="qName">
+ /// The element's qualified (prefixed) name, or the
+ /// empty string is none is available. This method will
+ /// use the qName as a template for generating a prefix
+ /// if necessary, but it is not guaranteed to use the
+ /// same qName.
+ /// </param>
+ /// <param name="atts">
+ /// The element's attribute list (must not be null).
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the start tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.StartElement" />
+ public override void StartElement(string uri, string localName, string qName, IAttributes atts)
+ {
+ elementLevel++;
+ nsSupport.PushContext();
+ if (forceDTD && !hasOutputDTD)
+ {
+ StartDTD(localName ?? qName, "", "");
+ }
+ Write('<');
+ WriteName(uri, localName, qName, true);
+ WriteAttributes(atts);
+ if (elementLevel == 1)
+ {
+ ForceNSDecls();
+ }
+ WriteNSDecls();
+ Write('>');
+ // System.out.println("%%%% startElement [" + qName + "] htmlMode = " + htmlMode);
+ if (htmlMode && (qName.Equals("script") || qName.Equals("style")))
+ {
+ cdataElement = true;
+ // System.out.println("%%%% CDATA element");
+ }
+ base.StartElement(uri, localName, qName, atts);
+ }
+
+ /// <summary>
+ /// Write an end tag.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI, or the empty string if none
+ /// is available.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local (unprefixed) name (required).
+ /// </param>
+ /// <param name="qName">
+ /// The element's qualified (prefixed) name, or the
+ /// empty string is none is available. This method will
+ /// use the qName as a template for generating a prefix
+ /// if necessary, but it is not guaranteed to use the
+ /// same qName.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the end tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.EndElement" />
+ public override void EndElement(string uri, string localName, string qName)
+ {
+ if (
+ !(htmlMode && (uri.Equals("http://www.w3.org/1999/xhtml") || uri.Equals(""))
+ && (qName.Equals("area") || qName.Equals("base") || qName.Equals("basefont") || qName.Equals("br")
+ || qName.Equals("col") || qName.Equals("frame") || qName.Equals("hr") || qName.Equals("img")
+ || qName.Equals("input") || qName.Equals("isindex") || qName.Equals("link") || qName.Equals("meta")
+ || qName.Equals("param"))))
+ {
+ Write("</");
+ WriteName(uri, localName, qName, true);
+ Write('>');
+ }
+ if (elementLevel == 1)
+ {
+ Write('\n');
+ }
+ cdataElement = false;
+ base.EndElement(uri, localName, qName);
+ nsSupport.PopContext();
+ elementLevel--;
+ }
+
+ /// <summary>
+ /// Write character data.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <param name="ch">
+ /// The array of characters to write.
+ /// </param>
+ /// <param name="start">
+ /// The starting position in the array.
+ /// </param>
+ /// <param name="length">
+ /// The number of characters to write.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the characters, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.Characters" />
+ public override void Characters(char[] ch, int start, int length)
+ {
+ if (!cdataElement)
+ {
+ WriteEsc(ch, start, length, false);
+ }
+ else
+ {
+ for (int i = start; i < start + length; i++)
+ {
+ Write(ch[i]);
+ }
+ }
+ base.Characters(ch, start, length);
+ }
+
+ /// <summary>
+ /// Write ignorable whitespace.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <param name="ch">
+ /// The array of characters to write.
+ /// </param>
+ /// <param name="start">
+ /// The starting position in the array.
+ /// </param>
+ /// <param name="length">
+ /// The number of characters to write.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the whitespace, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.IgnorableWhitespace" />
+ public override void IgnorableWhitespace(char[] ch, int start, int length)
+ {
+ WriteEsc(ch, start, length, false);
+ base.IgnorableWhitespace(ch, start, length);
+ }
+
+ /// <summary>
+ /// Write a processing instruction.
+ /// Pass the event on down the filter chain for further processing.
+ /// </summary>
+ /// <param name="target">
+ /// The PI target.
+ /// </param>
+ /// <param name="data">
+ /// The PI data.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the PI, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="IContentHandler.ProcessingInstruction" />
+ public override void ProcessingInstruction(string target, string data)
+ {
+ Write("<?");
+ Write(target);
+ Write(' ');
+ Write(data);
+ Write("?>");
+ if (elementLevel < 1)
+ {
+ Write('\n');
+ }
+ base.ProcessingInstruction(target, data);
+ }
+
+ /// <summary>
+ /// Write an empty element.
+ /// This method writes an empty element tag rather than a start tag
+ /// followed by an end tag. Both a <see cref="StartElement" />
+ /// and an <see cref="EndElement(string,string,string)" /> event will
+ /// be passed on down the filter chain.
+ /// </summary>
+ /// <param name="uri">
+ /// The element's Namespace URI, or the empty string
+ /// if the element has no Namespace or if Namespace
+ /// processing is not being performed.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local name (without prefix). This
+ /// parameter must be provided.
+ /// </param>
+ /// <param name="qName">
+ /// The element's qualified name (with prefix), or
+ /// the empty string if none is available. This parameter
+ /// is strictly advisory: the writer may or may not use
+ /// the prefix attached.
+ /// </param>
+ /// <param name="atts">
+ /// The element's attribute list.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the empty tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="StartElement" />
+ /// <seealso cref="EndElement(string,string,string) " />
+ public virtual void EmptyElement(string uri, string localName, string qName, IAttributes atts)
+ {
+ nsSupport.PushContext();
+ Write('<');
+ WriteName(uri, localName, qName, true);
+ WriteAttributes(atts);
+ if (elementLevel == 1)
+ {
+ ForceNSDecls();
+ }
+ WriteNSDecls();
+ Write("/>");
+ base.StartElement(uri, localName, qName, atts);
+ base.EndElement(uri, localName, qName);
+ }
+
+ /// <summary>
+ /// Start a new element without a qname or attributes.
+ /// <para>
+ /// This method will provide a default empty attribute
+ /// list and an empty string for the qualified name.
+ /// It invokes <see cref="StartElement(string, string, string, IAttributes)"/>
+ /// directly.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The element's Namespace URI.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the start tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="StartElement(string, string, string, IAttributes)" />
+ public virtual void StartElement(string uri, string localName)
+ {
+ StartElement(uri, localName, "", EMPTY_ATTS);
+ }
+
+ /// <summary>
+ /// Start a new element without a qname, attributes or a Namespace URI.
+ /// <para>
+ /// This method will provide an empty string for the
+ /// Namespace URI, and empty string for the qualified name,
+ /// and a default empty attribute list. It invokes
+ /// #startElement(string, string, string, Attributes)}
+ /// directly.
+ /// </para>
+ /// </summary>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the start tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="StartElement(string, string, string, IAttributes)" />
+ public virtual void StartElement(string localName)
+ {
+ StartElement("", localName, "", EMPTY_ATTS);
+ }
+
+ /// <summary>
+ /// End an element without a qname.
+ /// <para>
+ /// This method will supply an empty string for the qName.
+ /// It invokes <see cref="EndElement(string, string, string)" />
+ /// directly.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The element's Namespace URI.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the end tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="EndElement(string, string, string)" />
+ public virtual void EndElement(string uri, string localName)
+ {
+ EndElement(uri, localName, "");
+ }
+
+ /// <summary>
+ /// End an element without a Namespace URI or qname.
+ /// <para>
+ /// This method will supply an empty string for the qName
+ /// and an empty string for the Namespace URI.
+ /// It invokes <see cref="EndElement(string, string, string)" />
+ /// directly.
+ /// </para>
+ /// </summary>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the end tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="EndElement(string, string, string)" />
+ public virtual void EndElement(string localName)
+ {
+ EndElement("", localName, "");
+ }
+
+ /// <summary>
+ /// Add an empty element without a qname or attributes.
+ /// <para>
+ /// This method will supply an empty string for the qname
+ /// and an empty attribute list. It invokes
+ /// <see cref="EmptyElement(string, string, string, IAttributes)" />
+ /// directly.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The element's Namespace URI.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the empty tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="EmptyElement(string, string, string, IAttributes)" />
+ public virtual void EmptyElement(string uri, string localName)
+ {
+ EmptyElement(uri, localName, "", EMPTY_ATTS);
+ }
+
+ /// <summary>
+ /// Add an empty element without a Namespace URI, qname or attributes.
+ /// <para>
+ /// This method will supply an empty string for the qname,
+ /// and empty string for the Namespace URI, and an empty
+ /// attribute list. It invokes
+ /// <see cref="EmptyElement(string, string, string, IAttributes)" />
+ /// directly.
+ /// </para>
+ /// </summary>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the empty tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="EmptyElement(string, string, string, IAttributes)" />
+ public virtual void EmptyElement(string localName)
+ {
+ EmptyElement("", localName, "", EMPTY_ATTS);
+ }
+
+ /// <summary>
+ /// Write an element with character data content.
+ /// <para>
+ /// This is a convenience method to write a complete element
+ /// with character data content, including the start tag
+ /// and end tag.
+ /// </para>
+ /// <para>
+ /// This method invokes
+ /// <see cref="StartElement(string, string, string, IAttributes)" />,
+ /// followed by
+ /// <see cref="Characters(string)" />, followed by
+ /// <see cref="EndElement(string, string, string)" />.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The element's Namespace URI.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <param name="qName">
+ /// The element's default qualified name.
+ /// </param>
+ /// <param name="atts">
+ /// The element's attributes.
+ /// </param>
+ /// <param name="content">
+ /// The character data content.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the empty tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="StartElement(string, string, string, IAttributes)" />
+ /// <seealso cref="Characters(string)" />
+ /// <seealso cref="EndElement(string, string, string)" />
+ public virtual void DataElement(string uri, string localName, string qName, IAttributes atts, string content)
+ {
+ StartElement(uri, localName, qName, atts);
+ Characters(content);
+ EndElement(uri, localName, qName);
+ }
+
+ /// <summary>
+ /// Write an element with character data content but no attributes.
+ /// <para>
+ /// This is a convenience method to write a complete element
+ /// with character data content, including the start tag
+ /// and end tag. This method provides an empty string
+ /// for the qname and an empty attribute list.
+ /// </para>
+ /// <para>
+ /// This method invokes
+ /// <see cref="StartElement(string, string, string, IAttributes)" />,
+ /// followed by
+ /// <see cref="Characters(string)" />, followed by
+ /// <see cref="EndElement(string, string, string)" />.
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The element's Namespace URI.
+ /// </param>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <param name="content">
+ /// The character data content.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the empty tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="StartElement(string, string, string, IAttributes)" />
+ /// <seealso cref="Characters(string)" />
+ /// <seealso cref="EndElement(string, string, string)" />
+ public virtual void DataElement(string uri, string localName, string content)
+ {
+ DataElement(uri, localName, "", EMPTY_ATTS, content);
+ }
+
+ /// <summary>
+ /// Write an element with character data content but no attributes or Namespace URI.
+ /// <para>
+ /// This is a convenience method to write a complete element
+ /// with character data content, including the start tag
+ /// and end tag. The method provides an empty string for the
+ /// Namespace URI, and empty string for the qualified name,
+ /// and an empty attribute list.
+ /// </para>
+ /// <para>
+ /// This method invokes
+ /// <see cref="StartElement(string, string, string, IAttributes)" />,
+ /// followed by
+ /// <see cref="Characters(string)" />, followed by
+ /// <see cref="EndElement(string, string, string)" />.
+ /// </para>
+ /// </summary>
+ /// <param name="localName">
+ /// The element's local name.
+ /// </param>
+ /// <param name="content">
+ /// The character data content.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the empty tag, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="StartElement(string, string, string, IAttributes)" />
+ /// <seealso cref="Characters(string)" />
+ /// <seealso cref="EndElement(string, string, string)" />
+ public virtual void DataElement(string localName, string content)
+ {
+ DataElement("", localName, "", EMPTY_ATTS, content);
+ }
+
+ /// <summary>
+ /// Write a string of character data, with XML escaping.
+ /// <para>
+ /// This is a convenience method that takes an XML
+ /// string, converts it to a character array, then invokes
+ /// <see cref="Characters(char[], int, int)" />.
+ /// </para>
+ /// </summary>
+ /// <param name="data">
+ /// The character data.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error
+ /// writing the string, or if a handler further down
+ /// the filter chain raises an exception.
+ /// </exception>
+ /// <seealso cref="Characters(char[], int, int)" />
+ public virtual void Characters(string data)
+ {
+ char[] ch = data.ToCharArray();
+ Characters(ch, 0, ch.Length);
+ }
+
+ /// <summary>
+ /// Force all Namespaces to be declared.
+ /// This method is used on the root element to ensure that
+ /// the predeclared Namespaces all appear.
+ /// </summary>
+ private void ForceNSDecls()
+ {
+ foreach (string prefix in forcedDeclTable.Keys)
+ {
+ DoPrefix(prefix, null, true);
+ }
+ }
+
+ /// <summary>
+ /// Determine the prefix for an element or attribute name.
+ /// TODO: this method probably needs some cleanup.
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI.
+ /// </param>
+ /// <param name="qName">
+ /// The qualified name (optional); this will be used
+ /// to indicate the preferred prefix if none is currently
+ /// bound.
+ /// </param>
+ /// <param name="isElement">
+ /// true if this is an element name, false
+ /// if it is an attribute name (which cannot use the
+ /// default Namespace).
+ /// </param>
+ private string DoPrefix(string uri, string qName, bool isElement)
+ {
+ string defaultNS = nsSupport.GetUri("");
+ if ("".Equals(uri))
+ {
+ if (isElement && defaultNS != null)
+ {
+ nsSupport.DeclarePrefix("", "");
+ }
+ return null;
+ }
+ string prefix;
+ if (isElement && defaultNS != null && uri.Equals(defaultNS))
+ {
+ prefix = "";
+ }
+ else
+ {
+ prefix = nsSupport.GetPrefix(uri);
+ }
+ if (prefix != null)
+ {
+ return prefix;
+ }
+ bool containsPrefix = doneDeclTable.ContainsKey(uri);
+ prefix = (string)(containsPrefix ? doneDeclTable[uri] : null);
+ if (containsPrefix && ((!isElement || defaultNS != null) && "".Equals(prefix) || nsSupport.GetUri(prefix) != null))
+ {
+ prefix = null;
+ }
+ if (prefix == null)
+ {
+ containsPrefix = prefixTable.ContainsKey(uri);
+ prefix = (string)(containsPrefix ? prefixTable[uri] : null);
+ if (containsPrefix
+ && ((!isElement || defaultNS != null) && "".Equals(prefix) || nsSupport.GetUri(prefix) != null))
+ {
+ prefix = null;
+ }
+ }
+ if (prefix == null && qName != null && !"".Equals(qName))
+ {
+ int i = qName.IndexOf(':');
+ if (i == -1)
+ {
+ if (isElement && defaultNS == null)
+ {
+ prefix = "";
+ }
+ }
+ else
+ {
+ prefix = qName.Substring(0, i);
+ }
+ }
+ for (; prefix == null || nsSupport.GetUri(prefix) != null; prefix = "__NS" + ++prefixCounter)
+ {
+ }
+ nsSupport.DeclarePrefix(prefix, uri);
+ doneDeclTable[uri] = prefix;
+ return prefix;
+ }
+
+ /// <summary>
+ /// Write a raw character.
+ /// </summary>
+ /// <param name="c">
+ /// The character to write.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error writing
+ /// the character, this method will throw an IOException
+ /// wrapped in a SAXException.
+ /// </exception>
+ private void Write(char c)
+ {
+ try
+ {
+ output.Write(c);
+ }
+ catch (IOException e)
+ {
+ throw new SAXException(e.ToString(), e);
+ }
+ }
+
+ /// <summary>
+ /// Write a raw string.
+ /// </summary>
+ /// <param name="s"></param>
+ /// <exception cref="SAXException">
+ /// If there is an error writing the string,
+ /// this method will throw an IOException wrapped in a SAXException
+ /// </exception>
+ private void Write(string s)
+ {
+ try
+ {
+ output.Write(s);
+ }
+ catch (IOException e)
+ {
+ throw new SAXException(e.ToString(), e);
+ }
+ }
+
+ /// <summary>
+ /// Write out an attribute list, escaping values.
+ /// The names will have prefixes added to them.
+ /// </summary>
+ /// <param name="atts">
+ /// The attribute list to write.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error writing
+ /// the attribute list, this method will throw an
+ /// IOException wrapped in a SAXException.
+ /// </exception>
+ private void WriteAttributes(IAttributes atts)
+ {
+ int len = atts.Length;
+ for (int i = 0; i < len; i++)
+ {
+ char[] ch = atts.GetValue(i).ToCharArray();
+ Write(' ');
+ WriteName(atts.GetURI(i), atts.GetLocalName(i), atts.GetQName(i), false);
+ if (htmlMode && BoolAttribute(atts.GetLocalName(i), atts.GetQName(i), atts.GetValue(i)))
+ {
+ break;
+ }
+ Write("=\"");
+ WriteEsc(ch, 0, ch.Length, true);
+ Write('"');
+ }
+ }
+
+ // Return true if the attribute is an HTML bool from the above list.
+ private bool BoolAttribute(string localName, string qName, string value)
+ {
+ string name = localName;
+ if (name == null)
+ {
+ int i = qName.IndexOf(':');
+ if (i != -1)
+ {
+ name = qName.Substring(i + 1, qName.Length);
+ }
+ }
+ if (!name.Equals(value))
+ {
+ return false;
+ }
+ for (int j = 0; j < _bools.Length; j++)
+ {
+ if (name.Equals(_bools[j]))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /// <summary>
+ /// Write an array of data characters with escaping.
+ /// </summary>
+ /// <param name="ch">
+ /// The array of characters.
+ /// </param>
+ /// <param name="start">
+ /// The starting position.
+ /// </param>
+ /// <param name="length">
+ /// The number of characters to use.
+ /// </param>
+ /// <param name="isAttVal">
+ /// true if this is an attribute value literal.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// If there is an error writing
+ /// the characters, this method will throw an
+ /// IOException wrapped in a SAXException.
+ /// </exception>
+ private void WriteEsc(char[] ch, int start, int length, bool isAttVal)
+ {
+ for (int i = start; i < start + length; i++)
+ {
+ switch (ch[i])
+ {
+ case '&':
+ Write("&");
+ break;
+ case '<':
+ Write("<");
+ break;
+ case '>':
+ Write(">");
+ break;
+ case '\"':
+ if (isAttVal)
+ {
+ Write(""");
+ }
+ else
+ {
+ Write('\"');
+ }
+ break;
+ default:
+ if (!unicodeMode && ch[i] > '\u007f')
+ {
+ Write("&#");
+ Write(((int)ch[i]).ToString(CultureInfo.InvariantCulture));
+ Write(';');
+ }
+ else
+ {
+ Write(ch[i]);
+ }
+ break;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Write out the list of Namespace declarations.
+ /// </summary>
+ /// <exception cref="SAXException">
+ /// This method will throw
+ /// an IOException wrapped in a SAXException if
+ /// there is an error writing the Namespace
+ /// declarations.
+ /// </exception>
+ private void WriteNSDecls()
+ {
+ IEnumerable prefixes = nsSupport.GetDeclaredPrefixes();
+ foreach (string prefix in prefixes)
+ {
+ string uri = nsSupport.GetUri(prefix);
+ if (uri == null)
+ {
+ uri = "";
+ }
+ char[] ch = uri.ToCharArray();
+ Write(' ');
+ if ("".Equals(prefix))
+ {
+ Write("xmlns=\"");
+ }
+ else
+ {
+ Write("xmlns:");
+ Write(prefix);
+ Write("=\"");
+ }
+ WriteEsc(ch, 0, ch.Length, true);
+ Write('\"');
+ }
+ }
+
+ /// <summary>
+ /// Write an element or attribute name.
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI.
+ /// </param>
+ /// <param name="localName">
+ /// The local name.
+ /// </param>
+ /// <param name="qName">
+ /// The prefixed name, if available, or the empty string.
+ /// </param>
+ /// <param name="isElement">
+ /// true if this is an element name, false if it
+ /// is an attribute name.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// This method will throw an
+ /// IOException wrapped in a SAXException if there is
+ /// an error writing the name.
+ /// </exception>
+ private void WriteName(string uri, string localName, string qName, bool isElement)
+ {
+ string prefix = DoPrefix(uri, qName, isElement);
+ if (prefix != null && !"".Equals(prefix))
+ {
+ Write(prefix);
+ Write(':');
+ }
+ if (localName != null && !"".Equals(localName))
+ {
+ Write(localName);
+ }
+ else
+ {
+ int i = qName.IndexOf(':');
+ Write(qName.Substring(i + 1, qName.Length - (i + 1)));
+ }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Default LexicalHandler implementation
+ ////////////////////////////////////////////////////////////////////
+
+ public virtual void Comment(char[] ch, int start, int length)
+ {
+ Write("<!--");
+ for (int i = start; i < start + length; i++)
+ {
+ Write(ch[i]);
+ if (ch[i] == '-' && i + 1 <= start + length && ch[i + 1] == '-')
+ {
+ Write(' ');
+ }
+ }
+ Write("-->");
+ }
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Output properties
+ ////////////////////////////////////////////////////////////////////
+
+ public virtual string GetOutputProperty(string key)
+ {
+ return outputProperties[key];
+ }
+
+ public virtual void SetOutputProperty(string key, string value)
+ {
+ outputProperties[key] = value;
+ // System.out.println("%%%% key = [" + key + "] value = [" + value +"]");
+ if (key.Equals(ENCODING))
+ {
+ outputEncoding = value;
+ unicodeMode = value.Substring(0, 3).Equals("utf", StringComparison.OrdinalIgnoreCase);
+ // System.out.println("%%%% unicodeMode = " + unicodeMode);
+ }
+ else if (key.Equals(METHOD))
+ {
+ htmlMode = value.Equals("html");
+ }
+ else if (key.Equals(DOCTYPE_PUBLIC))
+ {
+ overridePublic = value;
+ forceDTD = true;
+ }
+ else if (key.Equals(DOCTYPE_SYSTEM))
+ {
+ overrideSystem = value;
+ forceDTD = true;
+ }
+ else if (key.Equals(VERSION))
+ {
+ version = value;
+ }
+ else if (key.Equals(STANDALONE))
+ {
+ standalone = value;
+ }
+ // System.out.println("%%%% htmlMode = " + htmlMode);
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Constants.
+ ////////////////////////////////////////////////////////////////////
+
+ private readonly IAttributes EMPTY_ATTS = new Attributes();
+ public const string CDATA_SECTION_ELEMENTS =
+ "cdata-section-elements";
+ public const string DOCTYPE_PUBLIC = "doctype-public";
+ public const string DOCTYPE_SYSTEM = "doctype-system";
+ public const string ENCODING = "encoding";
+ public const string INDENT = "indent"; // currently ignored
+ public const string MEDIA_TYPE = "media-type"; // currently ignored
+ public const string METHOD = "method"; // currently html or xml
+ public const string OMIT_XML_DECLARATION = "omit-xml-declaration";
+ public const string STANDALONE = "standalone"; // currently ignored
+ public const string VERSION = "version";
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////////
+
+
+ private readonly string[] _bools = {
+ "checked",
+ "compact",
+ "declare",
+ "defer",
+ "disabled",
+ "ismap",
+ "multiple",
+ "nohref",
+ "noresize",
+ "noshade",
+ "nowrap",
+ "readonly",
+ "selected"
+ };
+
+ private Hashtable prefixTable;
+ private Hashtable forcedDeclTable;
+ private Hashtable doneDeclTable;
+ private int elementLevel = 0;
+ private TextWriter output;
+ private NamespaceSupport nsSupport;
+ private int prefixCounter = 0;
+ private IDictionary<string, string> outputProperties;
+ private bool unicodeMode = false;
+ private string outputEncoding = "";
+ private bool htmlMode = false;
+ private bool forceDTD = false;
+ private bool hasOutputDTD = false;
+ private string overridePublic = null;
+ private string overrideSystem = null;
+ private string version = null;
+ private string standalone = null;
+ private bool cdataElement = false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.stml
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.stml b/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.stml
new file mode 100644
index 0000000..4cab973
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.stml
@@ -0,0 +1,249 @@
+<!--
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+-->
+
+<statetable xmlns='http://www.ccil.org/~cowan/XML/tagsoup/stml'
+ version='1.0'>
+
+ <symbol id='EOF'/>
+ <symbol id='S'/>
+ <symbol id='default'/>
+
+ <action id='A_ADUP'/>
+ <action id='A_ADUP_SAVE'/>
+ <action id='A_ADUP_STAGC'/>
+ <action id='A_ANAME'/>
+ <action id='A_ANAME_ADUP_STAGC'/>
+ <action id='A_AVAL'/>
+ <action id='A_AVAL_STAGC'/>
+ <action id='A_CDATA'/>
+ <action id='A_CMNT'/>
+ <action id='A_DECL'/>
+ <action id='A_ENTITY'/>
+ <action id='A_ENTITY_START'/>
+ <action id='A_ETAG'/>
+ <action id='A_EMPTYTAG'/>
+ <action id='A_ANAME_ADUP'/>
+ <action id='A_GI'/>
+ <action id='A_GI_STAGC'/>
+ <action id='A_LT'/>
+ <action id='A_LT_PCDATA'/>
+ <action id='A_MINUS'/>
+ <action id='A_MINUS2'/>
+ <action id='A_MINUS3'/>
+ <action id='A_PCDATA'/>
+ <action id='A_PI'/>
+ <action id='A_PITARGET'/>
+ <action id='A_PITARGET_PI'/>
+ <action id='A_SAVE'/>
+ <action id='A_SKIP'/>
+ <action id='A_SP'/>
+ <action id='A_STAGC'/>
+ <action id='A_UNGET'/>
+ <action id='A_UNSAVE_PCDATA'/>
+
+ <state id='S_ANAME'>
+ <tr symbol='default' action='A_SAVE' newstate='S_ANAME'/>
+ <tr char='=' action='A_ANAME' newstate='S_AVAL'/>
+ <tr char='>' action='A_ANAME_ADUP_STAGC' newstate='S_PCDATA'/>
+ <tr char='/' action='A_ANAME_ADUP' newstate='S_EMPTYTAG'/>
+ <tr symbol='EOF' action='A_ANAME_ADUP_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_ANAME' newstate='S_EQ'/>
+ </state>
+ <state id='S_APOS'>
+ <tr symbol='default' action='A_SAVE' newstate='S_APOS'/>
+ <tr char=''' action='A_AVAL' newstate='S_TAGWS'/>
+ <tr symbol='EOF' action='A_AVAL_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_SP' newstate='S_APOS'/>
+ </state>
+ <state id='S_AVAL'>
+ <tr symbol='default' action='A_SAVE' newstate='S_STAGC'/>
+ <tr char='"' action='A_SKIP' newstate='S_QUOT'/>
+ <tr char=''' action='A_SKIP' newstate='S_APOS'/>
+ <tr char='>' action='A_AVAL_STAGC' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_AVAL_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_SKIP' newstate='S_AVAL'/>
+ </state>
+ <state id='S_CDATA'>
+ <tr symbol='default' action='A_SAVE' newstate='S_CDATA'/>
+ <tr char='<' action='A_SAVE' newstate='S_CDATA2'/>
+ <tr symbol='EOF' action='A_PCDATA' newstate='S_DONE'/>
+ </state>
+ <state id='S_CDATA2'>
+ <tr symbol='default' action='A_SAVE' newstate='S_CDATA'/>
+ <tr char='/' action='A_UNSAVE_PCDATA' newstate='S_ETAG'/>
+ <tr symbol='EOF' action='A_UNSAVE_PCDATA' newstate='S_DONE'/>
+ </state>
+ <state id='S_COM'>
+ <tr symbol='default' action='A_SAVE' newstate='S_COM2'/>
+ <tr char='-' action='A_SKIP' newstate='S_COM2'/>
+ <tr symbol='EOF' action='A_CMNT' newstate='S_DONE'/>
+ </state>
+ <state id='S_COM2'>
+ <tr symbol='default' action='A_SAVE' newstate='S_COM2'/>
+ <tr char='-' action='A_SKIP' newstate='S_COM3'/>
+ <tr symbol='EOF' action='A_CMNT' newstate='S_DONE'/>
+ </state>
+ <state id='S_COM3'>
+ <tr symbol='default' action='A_MINUS' newstate='S_COM2'/>
+ <tr char='-' action='A_SKIP' newstate='S_COM4'/>
+ <tr symbol='EOF' action='A_CMNT' newstate='S_DONE'/>
+ </state>
+ <state id='S_COM4'>
+ <tr symbol='default' action='A_MINUS2' newstate='S_COM2'/>
+ <tr char='-' action='A_MINUS3' newstate='S_COM4'/>
+ <tr char='>' action='A_CMNT' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_CMNT' newstate='S_DONE'/>
+ </state>
+ <state id='S_DECL'>
+ <tr symbol='default' action='A_SAVE' newstate='S_DECL2'/>
+ <tr char='-' action='A_SKIP' newstate='S_COM'/>
+ <tr char='[' action='A_SKIP' newstate='S_BB'/>
+ <tr char='>' action='A_SKIP' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_DECL2'>
+ <tr symbol='default' action='A_SAVE' newstate='S_DECL2'/>
+ <tr char='>' action='A_DECL' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_ENT'>
+ <tr symbol='default' action='A_ENTITY' newstate='S_ENT'/>
+ <tr symbol='EOF' action='A_ENTITY' newstate='S_DONE'/>
+ </state>
+ <state id='S_EQ'>
+ <tr symbol='default' action='A_ADUP_SAVE' newstate='S_ANAME'/>
+ <tr char='=' action='A_SKIP' newstate='S_AVAL'/>
+ <tr char='>' action='A_ADUP_STAGC' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_ADUP_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_SKIP' newstate='S_EQ'/>
+ </state>
+ <state id='S_ETAG'>
+ <tr symbol='default' action='A_SAVE' newstate='S_ETAG'/>
+ <tr char='>' action='A_ETAG' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_ETAG' newstate='S_DONE'/>
+ <tr symbol='S' action='A_SKIP' newstate='S_ETAG'/>
+ </state>
+ <state id='S_GI'>
+ <tr symbol='default' action='A_SAVE' newstate='S_GI'/>
+ <tr char='/' action='A_SKIP' newstate='S_EMPTYTAG'/>
+ <tr char='>' action='A_GI_STAGC' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ <tr symbol='S' action='A_GI' newstate='S_TAGWS'/>
+ </state>
+ <state id='S_NCR'>
+ <tr symbol='default' action='A_ENTITY' newstate='S_NCR'/>
+ <tr symbol='EOF' action='A_ENTITY' newstate='S_DONE'/>
+ </state>
+ <state id='S_XNCR'>
+ <tr symbol='default' action='A_ENTITY' newstate='S_XNCR'/>
+ <tr symbol='EOF' action='A_ENTITY' newstate='S_DONE'/>
+ </state>
+ <state id='S_PCDATA'>
+ <tr symbol='default' action='A_SAVE' newstate='S_PCDATA'/>
+ <tr char='&' action='A_ENTITY_START' newstate='S_ENT'/>
+ <tr char='<' action='A_PCDATA' newstate='S_TAG'/>
+ <tr symbol='EOF' action='A_PCDATA' newstate='S_DONE'/>
+ </state>
+ <state id='S_PI'>
+ <tr symbol='default' action='A_SAVE' newstate='S_PI'/>
+ <tr char='>' action='A_PI' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_PI' newstate='S_DONE'/>
+ </state>
+ <state id='S_PITARGET'>
+ <tr symbol='default' action='A_SAVE' newstate='S_PITARGET'/>
+ <tr char='>' action='A_PITARGET_PI' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_PITARGET_PI' newstate='S_DONE'/>
+ <tr symbol='S' action='A_PITARGET' newstate='S_PI'/>
+ </state>
+ <state id='S_QUOT'>
+ <tr symbol='default' action='A_SAVE' newstate='S_QUOT'/>
+ <tr char='"' action='A_AVAL' newstate='S_TAGWS'/>
+ <tr symbol='EOF' action='A_AVAL_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_SP' newstate='S_QUOT'/>
+ </state>
+ <state id='S_STAGC'>
+ <tr symbol='default' action='A_SAVE' newstate='S_STAGC'/>
+ <tr char='>' action='A_AVAL_STAGC' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_AVAL_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_AVAL' newstate='S_TAGWS'/>
+ </state>
+ <state id='S_TAG'>
+ <tr symbol='default' action='A_SAVE' newstate='S_GI'/>
+ <tr char='!' action='A_SKIP' newstate='S_DECL'/>
+ <tr char='/' action='A_SKIP' newstate='S_ETAG'/>
+ <tr char='?' action='A_SKIP' newstate='S_PITARGET'/>
+ <tr char='<' action='A_SAVE' newstate='S_TAG'/>
+ <tr symbol='EOF' action='A_LT_PCDATA' newstate='S_DONE'/>
+ <tr symbol='S' action='A_LT' newstate='S_PCDATA'/>
+ </state>
+ <state id='S_TAGWS'>
+ <tr symbol='default' action='A_SAVE' newstate='S_ANAME'/>
+ <tr char='/' action='A_SKIP' newstate='S_EMPTYTAG'/>
+ <tr char='>' action='A_STAGC' newstate='S_PCDATA'/>
+ <tr symbol='EOF' action='A_STAGC' newstate='S_DONE'/>
+ <tr symbol='S' action='A_SKIP' newstate='S_TAGWS'/>
+ </state>
+ <state id='S_EMPTYTAG'>
+ <tr symbol='S' action='A_SKIP' newstate='S_TAGWS'/>
+ <tr symbol='default' action='A_SAVE' newstate='S_ANAME'/>
+ <tr char='>' action='A_EMPTYTAG' newstate='S_PCDATA'/>
+ </state>
+ <state id='S_BB'>
+ <tr char='C' action='A_SKIP' newstate='S_BBC'/>
+ <tr symbol='default' action='A_SKIP' newstate='S_DECL'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_BBC'>
+ <tr char='D' action='A_SKIP' newstate='S_BBCD'/>
+ <tr symbol='default' action='A_SKIP' newstate='S_DECL'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_BBCD'>
+ <tr char='A' action='A_SKIP' newstate='S_BBCDA'/>
+ <tr symbol='default' action='A_SKIP' newstate='S_DECL'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_BBCDA'>
+ <tr char='T' action='A_SKIP' newstate='S_BBCDAT'/>
+ <tr symbol='default' action='A_SKIP' newstate='S_DECL'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_BBCDAT'>
+ <tr char='A' action='A_SKIP' newstate='S_BBCDATA'/>
+ <tr symbol='default' action='A_SKIP' newstate='S_DECL'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_BBCDATA'>
+ <tr char='[' action='A_SKIP' newstate='S_CDSECT'/>
+ <tr symbol='default' action='A_SKIP' newstate='S_DECL'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_CDSECT'>
+ <tr char=']' action='A_SAVE' newstate='S_CDSECT1'/>
+ <tr symbol='default' action='A_SAVE' newstate='S_CDSECT'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_CDSECT1'>
+ <tr char=']' action='A_SAVE' newstate='S_CDSECT2'/>
+ <tr symbol='default' action='A_SAVE' newstate='S_CDSECT'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_CDSECT2'>
+ <tr char='>' action='A_CDATA' newstate='S_PCDATA'/>
+ <tr symbol='default' action='A_SAVE' newstate='S_CDSECT'/>
+ <tr symbol='EOF' action='A_SKIP' newstate='S_DONE'/>
+ </state>
+ <state id='S_DONE'/>
+</statetable>
[17/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.tssl
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.tssl b/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.tssl
new file mode 100644
index 0000000..7207862
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/definitions/html.tssl
@@ -0,0 +1,2762 @@
+<!--
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+-->
+
+<schema xmlns='http://www.ccil.org/~cowan/XML/tagsoup/tssl'
+ ns='http://www.w3.org/1999/xhtml' name='html'
+ prefix='html' version='1.0'>
+
+ <entity name='Aacgr' codepoint='0386'/>
+ <entity name='aacgr' codepoint='03AC'/>
+ <entity name='Aacute' codepoint='00C1'/>
+ <entity name='aacute' codepoint='00E1'/>
+ <entity name='Abreve' codepoint='0102'/>
+ <entity name='abreve' codepoint='0103'/>
+ <entity name='ac' codepoint='223E'/>
+ <entity name='acd' codepoint='223F'/>
+ <entity name='Acirc' codepoint='00C2'/>
+ <entity name='acirc' codepoint='00E2'/>
+ <entity name='acute' codepoint='00B4'/>
+ <entity name='Acy' codepoint='0410'/>
+ <entity name='acy' codepoint='0430'/>
+ <entity name='AElig' codepoint='00C6'/>
+ <entity name='aelig' codepoint='00E6'/>
+ <entity name='af' codepoint='2061'/>
+ <entity name='Afr' codepoint='1D504'/>
+ <entity name='afr' codepoint='1D51E'/>
+ <entity name='Agr' codepoint='0391'/>
+ <entity name='agr' codepoint='03B1'/>
+ <entity name='Agrave' codepoint='00C0'/>
+ <entity name='agrave' codepoint='00E0'/>
+ <entity name='alefsym' codepoint='2135'/>
+ <entity name='aleph' codepoint='2135'/>
+ <entity name='Alpha' codepoint='0391'/>
+ <entity name='alpha' codepoint='03B1'/>
+ <entity name='Amacr' codepoint='0100'/>
+ <entity name='amacr' codepoint='0101'/>
+ <entity name='amalg' codepoint='2A3F'/>
+ <entity name='amp' codepoint='0026'/>
+ <entity name='and' codepoint='2227'/>
+ <entity name='And' codepoint='2A53'/>
+ <entity name='andand' codepoint='2A55'/>
+ <entity name='andd' codepoint='2A5C'/>
+ <entity name='andslope' codepoint='2A58'/>
+ <entity name='andv' codepoint='2A5A'/>
+ <entity name='ang' codepoint='2220'/>
+ <entity name='ange' codepoint='29A4'/>
+ <entity name='angle' codepoint='2220'/>
+ <entity name='angmsd' codepoint='2221'/>
+ <entity name='angmsdaa' codepoint='29A8'/>
+ <entity name='angmsdab' codepoint='29A9'/>
+ <entity name='angmsdac' codepoint='29AA'/>
+ <entity name='angmsdad' codepoint='29AB'/>
+ <entity name='angmsdae' codepoint='29AC'/>
+ <entity name='angmsdaf' codepoint='29AD'/>
+ <entity name='angmsdag' codepoint='29AE'/>
+ <entity name='angmsdah' codepoint='29AF'/>
+ <entity name='angrt' codepoint='221F'/>
+ <entity name='angrtvb' codepoint='22BE'/>
+ <entity name='angrtvbd' codepoint='299D'/>
+ <entity name='angsph' codepoint='2222'/>
+ <entity name='angst' codepoint='212B'/>
+ <entity name='angzarr' codepoint='237C'/>
+ <entity name='Aogon' codepoint='0104'/>
+ <entity name='aogon' codepoint='0105'/>
+ <entity name='Aopf' codepoint='1D538'/>
+ <entity name='aopf' codepoint='1D552'/>
+ <entity name='ap' codepoint='2248'/>
+ <entity name='apacir' codepoint='2A6F'/>
+ <entity name='ape' codepoint='224A'/>
+ <entity name='apE' codepoint='2A70'/>
+ <entity name='apid' codepoint='224B'/>
+ <entity name='apos' codepoint='0027'/>
+ <entity name='ApplyFunction' codepoint='2061'/>
+ <entity name='approx' codepoint='2248'/>
+ <entity name='approxeq' codepoint='224A'/>
+ <entity name='Aring' codepoint='00C5'/>
+ <entity name='aring' codepoint='00E5'/>
+ <entity name='Ascr' codepoint='1D49C'/>
+ <entity name='ascr' codepoint='1D4B6'/>
+ <entity name='Assign' codepoint='2254'/>
+ <entity name='ast' codepoint='002A'/>
+ <entity name='asymp' codepoint='2248'/>
+ <entity name='asympeq' codepoint='224D'/>
+ <entity name='Atilde' codepoint='00C3'/>
+ <entity name='atilde' codepoint='00E3'/>
+ <entity name='Auml' codepoint='00C4'/>
+ <entity name='auml' codepoint='00E4'/>
+ <entity name='awconint' codepoint='2233'/>
+ <entity name='awint' codepoint='2A11'/>
+ <entity name='b.alpha' codepoint='1D6C2'/>
+ <entity name='b.beta' codepoint='1D6C3'/>
+ <entity name='b.chi' codepoint='1D6D8'/>
+ <entity name='b.Delta' codepoint='1D6AB'/>
+ <entity name='b.delta' codepoint='1D6C5'/>
+ <entity name='b.epsi' codepoint='1D6C6'/>
+ <entity name='b.epsiv' codepoint='1D6DC'/>
+ <entity name='b.eta' codepoint='1D6C8'/>
+ <entity name='b.Gamma' codepoint='1D6AA'/>
+ <entity name='b.gamma' codepoint='1D6C4'/>
+ <entity name='b.Gammad' codepoint='1D7CA'/>
+ <entity name='b.gammad' codepoint='1D7CB'/>
+ <entity name='b.iota' codepoint='1D6CA'/>
+ <entity name='b.kappa' codepoint='1D6CB'/>
+ <entity name='b.kappav' codepoint='1D6DE'/>
+ <entity name='b.Lambda' codepoint='1D6B2'/>
+ <entity name='b.lambda' codepoint='1D6CC'/>
+ <entity name='b.mu' codepoint='1D6CD'/>
+ <entity name='b.nu' codepoint='1D6CE'/>
+ <entity name='b.Omega' codepoint='1D6C0'/>
+ <entity name='b.omega' codepoint='1D6DA'/>
+ <entity name='b.Phi' codepoint='1D6BD'/>
+ <entity name='b.phi' codepoint='1D6D7'/>
+ <entity name='b.phiv' codepoint='1D6DF'/>
+ <entity name='b.Pi' codepoint='1D6B7'/>
+ <entity name='b.pi' codepoint='1D6D1'/>
+ <entity name='b.piv' codepoint='1D6E1'/>
+ <entity name='b.Psi' codepoint='1D6BF'/>
+ <entity name='b.psi' codepoint='1D6D9'/>
+ <entity name='b.rho' codepoint='1D6D2'/>
+ <entity name='b.rhov' codepoint='1D6E0'/>
+ <entity name='b.Sigma' codepoint='1D6BA'/>
+ <entity name='b.sigma' codepoint='1D6D4'/>
+ <entity name='b.sigmav' codepoint='1D6D3'/>
+ <entity name='b.tau' codepoint='1D6D5'/>
+ <entity name='b.Theta' codepoint='1D6AF'/>
+ <entity name='b.thetas' codepoint='1D6C9'/>
+ <entity name='b.thetav' codepoint='1D6DD'/>
+ <entity name='b.Upsi' codepoint='1D6BC'/>
+ <entity name='b.upsi' codepoint='1D6D6'/>
+ <entity name='b.Xi' codepoint='1D6B5'/>
+ <entity name='b.xi' codepoint='1D6CF'/>
+ <entity name='b.zeta' codepoint='1D6C7'/>
+ <entity name='backcong' codepoint='224C'/>
+ <entity name='backepsilon' codepoint='03F6'/>
+ <entity name='backprime' codepoint='2035'/>
+ <entity name='backsim' codepoint='223D'/>
+ <entity name='backsimeq' codepoint='22CD'/>
+ <entity name='Backslash' codepoint='2216'/>
+ <entity name='Barv' codepoint='2AE7'/>
+ <entity name='barvee' codepoint='22BD'/>
+ <entity name='barwed' codepoint='2305'/>
+ <entity name='Barwed' codepoint='2306'/>
+ <entity name='barwedge' codepoint='2305'/>
+ <entity name='bbrk' codepoint='23B5'/>
+ <entity name='bbrktbrk' codepoint='23B6'/>
+ <entity name='bcong' codepoint='224C'/>
+ <entity name='Bcy' codepoint='0411'/>
+ <entity name='bcy' codepoint='0431'/>
+ <entity name='bdquo' codepoint='201E'/>
+ <entity name='becaus' codepoint='2235'/>
+ <entity name='because' codepoint='2235'/>
+ <entity name='bemptyv' codepoint='29B0'/>
+ <entity name='bepsi' codepoint='03F6'/>
+ <entity name='bernou' codepoint='212C'/>
+ <entity name='Bernoullis' codepoint='212C'/>
+ <entity name='Beta' codepoint='0392'/>
+ <entity name='beta' codepoint='03B2'/>
+ <entity name='beth' codepoint='2136'/>
+ <entity name='between' codepoint='226C'/>
+ <entity name='Bfr' codepoint='1D505'/>
+ <entity name='bfr' codepoint='1D51F'/>
+ <entity name='Bgr' codepoint='0392'/>
+ <entity name='bgr' codepoint='03B2'/>
+ <entity name='bigcap' codepoint='22C2'/>
+ <entity name='bigcirc' codepoint='25EF'/>
+ <entity name='bigcup' codepoint='22C3'/>
+ <entity name='bigodot' codepoint='2A00'/>
+ <entity name='bigoplus' codepoint='2A01'/>
+ <entity name='bigotimes' codepoint='2A02'/>
+ <entity name='bigsqcup' codepoint='2A06'/>
+ <entity name='bigstar' codepoint='2605'/>
+ <entity name='bigtriangledown' codepoint='25BD'/>
+ <entity name='bigtriangleup' codepoint='25B3'/>
+ <entity name='biguplus' codepoint='2A04'/>
+ <entity name='bigvee' codepoint='22C1'/>
+ <entity name='bigwedge' codepoint='22C0'/>
+ <entity name='bkarow' codepoint='290D'/>
+ <entity name='blacklozenge' codepoint='29EB'/>
+ <entity name='blacksquare' codepoint='25AA'/>
+ <entity name='blacktriangle' codepoint='25B4'/>
+ <entity name='blacktriangledown' codepoint='25BE'/>
+ <entity name='blacktriangleleft' codepoint='25C2'/>
+ <entity name='blacktriangleright' codepoint='25B8'/>
+ <entity name='blank' codepoint='2423'/>
+ <entity name='blk12' codepoint='2592'/>
+ <entity name='blk14' codepoint='2591'/>
+ <entity name='blk34' codepoint='2593'/>
+ <entity name='block' codepoint='2588'/>
+ <entity name='bnot' codepoint='2310'/>
+ <entity name='bNot' codepoint='2AED'/>
+ <entity name='Bopf' codepoint='1D539'/>
+ <entity name='bopf' codepoint='1D553'/>
+ <entity name='bot' codepoint='22A5'/>
+ <entity name='bottom' codepoint='22A5'/>
+ <entity name='bowtie' codepoint='22C8'/>
+ <entity name='boxbox' codepoint='29C9'/>
+ <entity name='boxdl' codepoint='2510'/>
+ <entity name='boxdL' codepoint='2555'/>
+ <entity name='boxDl' codepoint='2556'/>
+ <entity name='boxDL' codepoint='2557'/>
+ <entity name='boxdr' codepoint='250C'/>
+ <entity name='boxdR' codepoint='2552'/>
+ <entity name='boxDr' codepoint='2553'/>
+ <entity name='boxDR' codepoint='2554'/>
+ <entity name='boxh' codepoint='2500'/>
+ <entity name='boxH' codepoint='2550'/>
+ <entity name='boxhd' codepoint='252C'/>
+ <entity name='boxHd' codepoint='2564'/>
+ <entity name='boxhD' codepoint='2565'/>
+ <entity name='boxHD' codepoint='2566'/>
+ <entity name='boxhu' codepoint='2534'/>
+ <entity name='boxHu' codepoint='2567'/>
+ <entity name='boxhU' codepoint='2568'/>
+ <entity name='boxHU' codepoint='2569'/>
+ <entity name='boxminus' codepoint='229F'/>
+ <entity name='boxplus' codepoint='229E'/>
+ <entity name='boxtimes' codepoint='22A0'/>
+ <entity name='boxul' codepoint='2518'/>
+ <entity name='boxuL' codepoint='255B'/>
+ <entity name='boxUl' codepoint='255C'/>
+ <entity name='boxUL' codepoint='255D'/>
+ <entity name='boxur' codepoint='2514'/>
+ <entity name='boxuR' codepoint='2558'/>
+ <entity name='boxUr' codepoint='2559'/>
+ <entity name='boxUR' codepoint='255A'/>
+ <entity name='boxv' codepoint='2502'/>
+ <entity name='boxV' codepoint='2551'/>
+ <entity name='boxvh' codepoint='253C'/>
+ <entity name='boxvH' codepoint='256A'/>
+ <entity name='boxVh' codepoint='256B'/>
+ <entity name='boxVH' codepoint='256C'/>
+ <entity name='boxvl' codepoint='2524'/>
+ <entity name='boxvL' codepoint='2561'/>
+ <entity name='boxVl' codepoint='2562'/>
+ <entity name='boxVL' codepoint='2563'/>
+ <entity name='boxvr' codepoint='251C'/>
+ <entity name='boxvR' codepoint='255E'/>
+ <entity name='boxVr' codepoint='255F'/>
+ <entity name='boxVR' codepoint='2560'/>
+ <entity name='bprime' codepoint='2035'/>
+ <entity name='breve' codepoint='02D8'/>
+ <entity name='brvbar' codepoint='00A6'/>
+ <entity name='Bscr' codepoint='212C'/>
+ <entity name='bscr' codepoint='1D4B7'/>
+ <entity name='bsemi' codepoint='204F'/>
+ <entity name='bsim' codepoint='223D'/>
+ <entity name='bsime' codepoint='22CD'/>
+ <entity name='bsol' codepoint='005C'/>
+ <entity name='bsolb' codepoint='29C5'/>
+ <entity name='bull' codepoint='2022'/>
+ <entity name='bullet' codepoint='2022'/>
+ <entity name='bump' codepoint='224E'/>
+ <entity name='bumpe' codepoint='224F'/>
+ <entity name='bumpE' codepoint='2AAE'/>
+ <entity name='Bumpeq' codepoint='224E'/>
+ <entity name='bumpeq' codepoint='224F'/>
+ <entity name='Cacute' codepoint='0106'/>
+ <entity name='cacute' codepoint='0107'/>
+ <entity name='cap' codepoint='2229'/>
+ <entity name='Cap' codepoint='22D2'/>
+ <entity name='capand' codepoint='2A44'/>
+ <entity name='capbrcup' codepoint='2A49'/>
+ <entity name='capcap' codepoint='2A4B'/>
+ <entity name='capcup' codepoint='2A47'/>
+ <entity name='capdot' codepoint='2A40'/>
+ <entity name='CapitalDifferentialD' codepoint='2145'/>
+ <entity name='caret' codepoint='2041'/>
+ <entity name='caron' codepoint='02C7'/>
+ <entity name='Cayleys' codepoint='212D'/>
+ <entity name='ccaps' codepoint='2A4D'/>
+ <entity name='Ccaron' codepoint='010C'/>
+ <entity name='ccaron' codepoint='010D'/>
+ <entity name='Ccedil' codepoint='00C7'/>
+ <entity name='ccedil' codepoint='00E7'/>
+ <entity name='Ccirc' codepoint='0108'/>
+ <entity name='ccirc' codepoint='0109'/>
+ <entity name='Cconint' codepoint='2230'/>
+ <entity name='ccups' codepoint='2A4C'/>
+ <entity name='ccupssm' codepoint='2A50'/>
+ <entity name='Cdot' codepoint='010A'/>
+ <entity name='cdot' codepoint='010B'/>
+ <entity name='cedil' codepoint='00B8'/>
+ <entity name='Cedilla' codepoint='00B8'/>
+ <entity name='cemptyv' codepoint='29B2'/>
+ <entity name='cent' codepoint='00A2'/>
+ <entity name='centerdot' codepoint='00B7'/>
+ <entity name='Cfr' codepoint='212D'/>
+ <entity name='cfr' codepoint='1D520'/>
+ <entity name='CHcy' codepoint='0427'/>
+ <entity name='chcy' codepoint='0447'/>
+ <entity name='check' codepoint='2713'/>
+ <entity name='checkmark' codepoint='2713'/>
+ <entity name='Chi' codepoint='03A7'/>
+ <entity name='chi' codepoint='03C7'/>
+ <entity name='cir' codepoint='25CB'/>
+ <entity name='circ' codepoint='02C6'/>
+ <entity name='circeq' codepoint='2257'/>
+ <entity name='circlearrowleft' codepoint='21BA'/>
+ <entity name='circlearrowright' codepoint='21BB'/>
+ <entity name='circledast' codepoint='229B'/>
+ <entity name='circledcirc' codepoint='229A'/>
+ <entity name='circleddash' codepoint='229D'/>
+ <entity name='CircleDot' codepoint='2299'/>
+ <entity name='circledR' codepoint='00AE'/>
+ <entity name='circledS' codepoint='24C8'/>
+ <entity name='CircleMinus' codepoint='2296'/>
+ <entity name='CirclePlus' codepoint='2295'/>
+ <entity name='CircleTimes' codepoint='2297'/>
+ <entity name='cire' codepoint='2257'/>
+ <entity name='cirE' codepoint='29C3'/>
+ <entity name='cirfnint' codepoint='2A10'/>
+ <entity name='cirmid' codepoint='2AEF'/>
+ <entity name='cirscir' codepoint='29C2'/>
+ <entity name='ClockwiseContourIntegral' codepoint='2232'/>
+ <entity name='CloseCurlyDoubleQuote' codepoint='201D'/>
+ <entity name='CloseCurlyQuote' codepoint='2019'/>
+ <entity name='clubs' codepoint='2663'/>
+ <entity name='clubsuit' codepoint='2663'/>
+ <entity name='colon' codepoint='003A'/>
+ <entity name='Colon' codepoint='2237'/>
+ <entity name='colone' codepoint='2254'/>
+ <entity name='Colone' codepoint='2A74'/>
+ <entity name='coloneq' codepoint='2254'/>
+ <entity name='comma' codepoint='002C'/>
+ <entity name='commat' codepoint='0040'/>
+ <entity name='comp' codepoint='2201'/>
+ <entity name='compfn' codepoint='2218'/>
+ <entity name='complement' codepoint='2201'/>
+ <entity name='complexes' codepoint='2102'/>
+ <entity name='cong' codepoint='2245'/>
+ <entity name='congdot' codepoint='2A6D'/>
+ <entity name='Congruent' codepoint='2261'/>
+ <entity name='conint' codepoint='222E'/>
+ <entity name='Conint' codepoint='222F'/>
+ <entity name='ContourIntegral' codepoint='222E'/>
+ <entity name='Copf' codepoint='2102'/>
+ <entity name='copf' codepoint='1D554'/>
+ <entity name='coprod' codepoint='2210'/>
+ <entity name='Coproduct' codepoint='2210'/>
+ <entity name='copy' codepoint='00A9'/>
+ <entity name='copysr' codepoint='2117'/>
+ <entity name='CounterClockwiseContourIntegral' codepoint='2233'/>
+ <entity name='crarr' codepoint='21B5'/>
+ <entity name='cross' codepoint='2717'/>
+ <entity name='Cross' codepoint='2A2F'/>
+ <entity name='Cscr' codepoint='1D49E'/>
+ <entity name='cscr' codepoint='1D4B8'/>
+ <entity name='csub' codepoint='2ACF'/>
+ <entity name='csube' codepoint='2AD1'/>
+ <entity name='csup' codepoint='2AD0'/>
+ <entity name='csupe' codepoint='2AD2'/>
+ <entity name='ctdot' codepoint='22EF'/>
+ <entity name='cudarrl' codepoint='2938'/>
+ <entity name='cudarrr' codepoint='2935'/>
+ <entity name='cuepr' codepoint='22DE'/>
+ <entity name='cuesc' codepoint='22DF'/>
+ <entity name='cularr' codepoint='21B6'/>
+ <entity name='cularrp' codepoint='293D'/>
+ <entity name='cup' codepoint='222A'/>
+ <entity name='Cup' codepoint='22D3'/>
+ <entity name='cupbrcap' codepoint='2A48'/>
+ <entity name='CupCap' codepoint='224D'/>
+ <entity name='cupcap' codepoint='2A46'/>
+ <entity name='cupcup' codepoint='2A4A'/>
+ <entity name='cupdot' codepoint='228D'/>
+ <entity name='cupor' codepoint='2A45'/>
+ <entity name='curarr' codepoint='21B7'/>
+ <entity name='curarrm' codepoint='293C'/>
+ <entity name='curlyeqprec' codepoint='22DE'/>
+ <entity name='curlyeqsucc' codepoint='22DF'/>
+ <entity name='curlyvee' codepoint='22CE'/>
+ <entity name='curlywedge' codepoint='22CF'/>
+ <entity name='curren' codepoint='00A4'/>
+ <entity name='curvearrowleft' codepoint='21B6'/>
+ <entity name='curvearrowright' codepoint='21B7'/>
+ <entity name='cuvee' codepoint='22CE'/>
+ <entity name='cuwed' codepoint='22CF'/>
+ <entity name='cwconint' codepoint='2232'/>
+ <entity name='cwint' codepoint='2231'/>
+ <entity name='cylcty' codepoint='232D'/>
+ <entity name='dagger' codepoint='2020'/>
+ <entity name='Dagger' codepoint='2021'/>
+ <entity name='daleth' codepoint='2138'/>
+ <entity name='darr' codepoint='2193'/>
+ <entity name='Darr' codepoint='21A1'/>
+ <entity name='dArr' codepoint='21D3'/>
+ <entity name='dash' codepoint='2010'/>
+ <entity name='dashv' codepoint='22A3'/>
+ <entity name='Dashv' codepoint='2AE4'/>
+ <entity name='dbkarow' codepoint='290F'/>
+ <entity name='dblac' codepoint='02DD'/>
+ <entity name='Dcaron' codepoint='010E'/>
+ <entity name='dcaron' codepoint='010F'/>
+ <entity name='Dcy' codepoint='0414'/>
+ <entity name='dcy' codepoint='0434'/>
+ <entity name='DD' codepoint='2145'/>
+ <entity name='dd' codepoint='2146'/>
+ <entity name='ddagger' codepoint='2021'/>
+ <entity name='ddarr' codepoint='21CA'/>
+ <entity name='DDotrahd' codepoint='2911'/>
+ <entity name='ddotseq' codepoint='2A77'/>
+ <entity name='deg' codepoint='00B0'/>
+ <entity name='Del' codepoint='2207'/>
+ <entity name='Delta' codepoint='0394'/>
+ <entity name='delta' codepoint='03B4'/>
+ <entity name='demptyv' codepoint='29B1'/>
+ <entity name='dfisht' codepoint='297F'/>
+ <entity name='Dfr' codepoint='1D507'/>
+ <entity name='dfr' codepoint='1D521'/>
+ <entity name='Dgr' codepoint='0394'/>
+ <entity name='dgr' codepoint='03B4'/>
+ <entity name='dHar' codepoint='2965'/>
+ <entity name='dharl' codepoint='21C3'/>
+ <entity name='dharr' codepoint='21C2'/>
+ <entity name='DiacriticalAcute' codepoint='00B4'/>
+ <entity name='DiacriticalDot' codepoint='02D9'/>
+ <entity name='DiacriticalDoubleAcute' codepoint='02DD'/>
+ <entity name='DiacriticalGrave' codepoint='0060'/>
+ <entity name='DiacriticalTilde' codepoint='02DC'/>
+ <entity name='diam' codepoint='22C4'/>
+ <entity name='diamond' codepoint='22C4'/>
+ <entity name='diamondsuit' codepoint='2666'/>
+ <entity name='diams' codepoint='2666'/>
+ <entity name='die' codepoint='00A8'/>
+ <entity name='DifferentialD' codepoint='2146'/>
+ <entity name='digamma' codepoint='03DD'/>
+ <entity name='disin' codepoint='22F2'/>
+ <entity name='div' codepoint='00F7'/>
+ <entity name='divide' codepoint='00F7'/>
+ <entity name='divideontimes' codepoint='22C7'/>
+ <entity name='divonx' codepoint='22C7'/>
+ <entity name='DJcy' codepoint='0402'/>
+ <entity name='djcy' codepoint='0452'/>
+ <entity name='dlcorn' codepoint='231E'/>
+ <entity name='dlcrop' codepoint='230D'/>
+ <entity name='dollar' codepoint='0024'/>
+ <entity name='Dopf' codepoint='1D53B'/>
+ <entity name='dopf' codepoint='1D555'/>
+ <entity name='Dot' codepoint='00A8'/>
+ <entity name='dot' codepoint='02D9'/>
+ <entity name='doteq' codepoint='2250'/>
+ <entity name='doteqdot' codepoint='2251'/>
+ <entity name='DotEqual' codepoint='2250'/>
+ <entity name='dotminus' codepoint='2238'/>
+ <entity name='dotplus' codepoint='2214'/>
+ <entity name='dotsquare' codepoint='22A1'/>
+ <entity name='doublebarwedge' codepoint='2306'/>
+ <entity name='DoubleContourIntegral' codepoint='222F'/>
+ <entity name='DoubleDot' codepoint='00A8'/>
+ <entity name='DoubleDownArrow' codepoint='21D3'/>
+ <entity name='DoubleLeftArrow' codepoint='21D0'/>
+ <entity name='DoubleLeftRightArrow' codepoint='21D4'/>
+ <entity name='DoubleLeftTee' codepoint='2AE4'/>
+ <entity name='DoubleLongLeftArrow' codepoint='27F8'/>
+ <entity name='DoubleLongLeftRightArrow' codepoint='27FA'/>
+ <entity name='DoubleLongRightArrow' codepoint='27F9'/>
+ <entity name='DoubleRightArrow' codepoint='21D2'/>
+ <entity name='DoubleRightTee' codepoint='22A8'/>
+ <entity name='DoubleUpArrow' codepoint='21D1'/>
+ <entity name='DoubleUpDownArrow' codepoint='21D5'/>
+ <entity name='DoubleVerticalBar' codepoint='2225'/>
+ <entity name='downarrow' codepoint='2193'/>
+ <entity name='Downarrow' codepoint='21D3'/>
+ <entity name='DownArrowBar' codepoint='2913'/>
+ <entity name='DownArrowUpArrow' codepoint='21F5'/>
+ <entity name='downdownarrows' codepoint='21CA'/>
+ <entity name='downharpoonleft' codepoint='21C3'/>
+ <entity name='downharpoonright' codepoint='21C2'/>
+ <entity name='DownLeftRightVector' codepoint='2950'/>
+ <entity name='DownLeftTeeVector' codepoint='295E'/>
+ <entity name='DownLeftVector' codepoint='21BD'/>
+ <entity name='DownLeftVectorBar' codepoint='2956'/>
+ <entity name='DownRightTeeVector' codepoint='295F'/>
+ <entity name='DownRightVector' codepoint='21C1'/>
+ <entity name='DownRightVectorBar' codepoint='2957'/>
+ <entity name='DownTee' codepoint='22A4'/>
+ <entity name='DownTeeArrow' codepoint='21A7'/>
+ <entity name='drbkarow' codepoint='2910'/>
+ <entity name='drcorn' codepoint='231F'/>
+ <entity name='drcrop' codepoint='230C'/>
+ <entity name='Dscr' codepoint='1D49F'/>
+ <entity name='dscr' codepoint='1D4B9'/>
+ <entity name='DScy' codepoint='0405'/>
+ <entity name='dscy' codepoint='0455'/>
+ <entity name='dsol' codepoint='29F6'/>
+ <entity name='Dstrok' codepoint='0110'/>
+ <entity name='dstrok' codepoint='0111'/>
+ <entity name='dtdot' codepoint='22F1'/>
+ <entity name='dtri' codepoint='25BF'/>
+ <entity name='dtrif' codepoint='25BE'/>
+ <entity name='duarr' codepoint='21F5'/>
+ <entity name='duhar' codepoint='296F'/>
+ <entity name='dwangle' codepoint='29A6'/>
+ <entity name='DZcy' codepoint='040F'/>
+ <entity name='dzcy' codepoint='045F'/>
+ <entity name='dzigrarr' codepoint='27FF'/>
+ <entity name='Eacgr' codepoint='0388'/>
+ <entity name='eacgr' codepoint='03AD'/>
+ <entity name='Eacute' codepoint='00C9'/>
+ <entity name='eacute' codepoint='00E9'/>
+ <entity name='easter' codepoint='2A6E'/>
+ <entity name='Ecaron' codepoint='011A'/>
+ <entity name='ecaron' codepoint='011B'/>
+ <entity name='ecir' codepoint='2256'/>
+ <entity name='Ecirc' codepoint='00CA'/>
+ <entity name='ecirc' codepoint='00EA'/>
+ <entity name='ecolon' codepoint='2255'/>
+ <entity name='Ecy' codepoint='042D'/>
+ <entity name='ecy' codepoint='044D'/>
+ <entity name='eDDot' codepoint='2A77'/>
+ <entity name='Edot' codepoint='0116'/>
+ <entity name='edot' codepoint='0117'/>
+ <entity name='eDot' codepoint='2251'/>
+ <entity name='ee' codepoint='2147'/>
+ <entity name='EEacgr' codepoint='0389'/>
+ <entity name='eeacgr' codepoint='03AE'/>
+ <entity name='EEgr' codepoint='0397'/>
+ <entity name='eegr' codepoint='03B7'/>
+ <entity name='efDot' codepoint='2252'/>
+ <entity name='Efr' codepoint='1D508'/>
+ <entity name='efr' codepoint='1D522'/>
+ <entity name='eg' codepoint='2A9A'/>
+ <entity name='Egr' codepoint='0395'/>
+ <entity name='egr' codepoint='03B5'/>
+ <entity name='Egrave' codepoint='00C8'/>
+ <entity name='egrave' codepoint='00E8'/>
+ <entity name='egs' codepoint='2A96'/>
+ <entity name='egsdot' codepoint='2A98'/>
+ <entity name='el' codepoint='2A99'/>
+ <entity name='Element' codepoint='2208'/>
+ <entity name='elinters' codepoint='23E7'/>
+ <entity name='ell' codepoint='2113'/>
+ <entity name='els' codepoint='2A95'/>
+ <entity name='elsdot' codepoint='2A97'/>
+ <entity name='Emacr' codepoint='0112'/>
+ <entity name='emacr' codepoint='0113'/>
+ <entity name='empty' codepoint='2205'/>
+ <entity name='emptyset' codepoint='2205'/>
+ <entity name='EmptySmallSquare' codepoint='25FB'/>
+ <entity name='emptyv' codepoint='2205'/>
+ <entity name='EmptyVerySmallSquare' codepoint='25AB'/>
+ <entity name='emsp' codepoint='2003'/>
+ <entity name='emsp13' codepoint='2004'/>
+ <entity name='emsp14' codepoint='2005'/>
+ <entity name='ENG' codepoint='014A'/>
+ <entity name='eng' codepoint='014B'/>
+ <entity name='ensp' codepoint='2002'/>
+ <entity name='Eogon' codepoint='0118'/>
+ <entity name='eogon' codepoint='0119'/>
+ <entity name='Eopf' codepoint='1D53C'/>
+ <entity name='eopf' codepoint='1D556'/>
+ <entity name='epar' codepoint='22D5'/>
+ <entity name='eparsl' codepoint='29E3'/>
+ <entity name='eplus' codepoint='2A71'/>
+ <entity name='epsi' codepoint='03F5'/>
+ <entity name='Epsilon' codepoint='0395'/>
+ <entity name='epsilon' codepoint='03B5'/>
+ <entity name='epsiv' codepoint='03B5'/>
+ <entity name='eqcirc' codepoint='2256'/>
+ <entity name='eqcolon' codepoint='2255'/>
+ <entity name='eqsim' codepoint='2242'/>
+ <entity name='eqslantgtr' codepoint='2A96'/>
+ <entity name='eqslantless' codepoint='2A95'/>
+ <entity name='Equal' codepoint='2A75'/>
+ <entity name='equals' codepoint='003D'/>
+ <entity name='EqualTilde' codepoint='2242'/>
+ <entity name='equest' codepoint='225F'/>
+ <entity name='Equilibrium' codepoint='21CC'/>
+ <entity name='equiv' codepoint='2261'/>
+ <entity name='equivDD' codepoint='2A78'/>
+ <entity name='eqvparsl' codepoint='29E5'/>
+ <entity name='erarr' codepoint='2971'/>
+ <entity name='erDot' codepoint='2253'/>
+ <entity name='escr' codepoint='212F'/>
+ <entity name='Escr' codepoint='2130'/>
+ <entity name='esdot' codepoint='2250'/>
+ <entity name='esim' codepoint='2242'/>
+ <entity name='Esim' codepoint='2A73'/>
+ <entity name='Eta' codepoint='0397'/>
+ <entity name='eta' codepoint='03B7'/>
+ <entity name='ETH' codepoint='00D0'/>
+ <entity name='eth' codepoint='00F0'/>
+ <entity name='Euml' codepoint='00CB'/>
+ <entity name='euml' codepoint='00EB'/>
+ <entity name='euro' codepoint='20AC'/>
+ <entity name='excl' codepoint='0021'/>
+ <entity name='exist' codepoint='2203'/>
+ <entity name='Exists' codepoint='2203'/>
+ <entity name='expectation' codepoint='2130'/>
+ <entity name='exponentiale' codepoint='2147'/>
+ <entity name='fallingdotseq' codepoint='2252'/>
+ <entity name='Fcy' codepoint='0424'/>
+ <entity name='fcy' codepoint='0444'/>
+ <entity name='female' codepoint='2640'/>
+ <entity name='ffilig' codepoint='FB03'/>
+ <entity name='fflig' codepoint='FB00'/>
+ <entity name='ffllig' codepoint='FB04'/>
+ <entity name='Ffr' codepoint='1D509'/>
+ <entity name='ffr' codepoint='1D523'/>
+ <entity name='filig' codepoint='FB01'/>
+ <entity name='FilledSmallSquare' codepoint='25FC'/>
+ <entity name='FilledVerySmallSquare' codepoint='25AA'/>
+ <entity name='flat' codepoint='266D'/>
+ <entity name='fllig' codepoint='FB02'/>
+ <entity name='fltns' codepoint='25B1'/>
+ <entity name='fnof' codepoint='0192'/>
+ <entity name='Fopf' codepoint='1D53D'/>
+ <entity name='fopf' codepoint='1D557'/>
+ <entity name='forall' codepoint='2200'/>
+ <entity name='fork' codepoint='22D4'/>
+ <entity name='forkv' codepoint='2AD9'/>
+ <entity name='Fouriertrf' codepoint='2131'/>
+ <entity name='fpartint' codepoint='2A0D'/>
+ <entity name='frac12' codepoint='00BD'/>
+ <entity name='frac13' codepoint='2153'/>
+ <entity name='frac14' codepoint='00BC'/>
+ <entity name='frac15' codepoint='2155'/>
+ <entity name='frac16' codepoint='2159'/>
+ <entity name='frac18' codepoint='215B'/>
+ <entity name='frac23' codepoint='2154'/>
+ <entity name='frac25' codepoint='2156'/>
+ <entity name='frac34' codepoint='00BE'/>
+ <entity name='frac35' codepoint='2157'/>
+ <entity name='frac38' codepoint='215C'/>
+ <entity name='frac45' codepoint='2158'/>
+ <entity name='frac56' codepoint='215A'/>
+ <entity name='frac58' codepoint='215D'/>
+ <entity name='frac78' codepoint='215E'/>
+ <entity name='frasl' codepoint='2044'/>
+ <entity name='frown' codepoint='2322'/>
+ <entity name='Fscr' codepoint='2131'/>
+ <entity name='fscr' codepoint='1D4BB'/>
+ <entity name='gacute' codepoint='01F5'/>
+ <entity name='Gamma' codepoint='0393'/>
+ <entity name='gamma' codepoint='03B3'/>
+ <entity name='Gammad' codepoint='03DC'/>
+ <entity name='gammad' codepoint='03DD'/>
+ <entity name='gap' codepoint='2A86'/>
+ <entity name='Gbreve' codepoint='011E'/>
+ <entity name='gbreve' codepoint='011F'/>
+ <entity name='Gcedil' codepoint='0122'/>
+ <entity name='Gcirc' codepoint='011C'/>
+ <entity name='gcirc' codepoint='011D'/>
+ <entity name='Gcy' codepoint='0413'/>
+ <entity name='gcy' codepoint='0433'/>
+ <entity name='Gdot' codepoint='0120'/>
+ <entity name='gdot' codepoint='0121'/>
+ <entity name='ge' codepoint='2265'/>
+ <entity name='gE' codepoint='2267'/>
+ <entity name='gel' codepoint='22DB'/>
+ <entity name='gEl' codepoint='2A8C'/>
+ <entity name='geq' codepoint='2265'/>
+ <entity name='geqq' codepoint='2267'/>
+ <entity name='geqslant' codepoint='2A7E'/>
+ <entity name='ges' codepoint='2A7E'/>
+ <entity name='gescc' codepoint='2AA9'/>
+ <entity name='gesdot' codepoint='2A80'/>
+ <entity name='gesdoto' codepoint='2A82'/>
+ <entity name='gesdotol' codepoint='2A84'/>
+ <entity name='gesles' codepoint='2A94'/>
+ <entity name='Gfr' codepoint='1D50A'/>
+ <entity name='gfr' codepoint='1D524'/>
+ <entity name='gg' codepoint='226B'/>
+ <entity name='Gg' codepoint='22D9'/>
+ <entity name='ggg' codepoint='22D9'/>
+ <entity name='Ggr' codepoint='0393'/>
+ <entity name='ggr' codepoint='03B3'/>
+ <entity name='gimel' codepoint='2137'/>
+ <entity name='GJcy' codepoint='0403'/>
+ <entity name='gjcy' codepoint='0453'/>
+ <entity name='gl' codepoint='2277'/>
+ <entity name='gla' codepoint='2AA5'/>
+ <entity name='glE' codepoint='2A92'/>
+ <entity name='glj' codepoint='2AA4'/>
+ <entity name='gnap' codepoint='2A8A'/>
+ <entity name='gnapprox' codepoint='2A8A'/>
+ <entity name='gnE' codepoint='2269'/>
+ <entity name='gne' codepoint='2A88'/>
+ <entity name='gneq' codepoint='2A88'/>
+ <entity name='gneqq' codepoint='2269'/>
+ <entity name='gnsim' codepoint='22E7'/>
+ <entity name='Gopf' codepoint='1D53E'/>
+ <entity name='gopf' codepoint='1D558'/>
+ <entity name='grave' codepoint='0060'/>
+ <entity name='GreaterEqual' codepoint='2265'/>
+ <entity name='GreaterEqualLess' codepoint='22DB'/>
+ <entity name='GreaterFullEqual' codepoint='2267'/>
+ <entity name='GreaterGreater' codepoint='2AA2'/>
+ <entity name='GreaterLess' codepoint='2277'/>
+ <entity name='GreaterSlantEqual' codepoint='2A7E'/>
+ <entity name='GreaterTilde' codepoint='2273'/>
+ <entity name='gscr' codepoint='210A'/>
+ <entity name='Gscr' codepoint='1D4A2'/>
+ <entity name='gsim' codepoint='2273'/>
+ <entity name='gsime' codepoint='2A8E'/>
+ <entity name='gsiml' codepoint='2A90'/>
+ <entity name='gt' codepoint='003E'/>
+ <entity name='Gt' codepoint='226B'/>
+ <entity name='gtcc' codepoint='2AA7'/>
+ <entity name='gtcir' codepoint='2A7A'/>
+ <entity name='gtdot' codepoint='22D7'/>
+ <entity name='gtlPar' codepoint='2995'/>
+ <entity name='gtquest' codepoint='2A7C'/>
+ <entity name='gtrapprox' codepoint='2A86'/>
+ <entity name='gtrarr' codepoint='2978'/>
+ <entity name='gtrdot' codepoint='22D7'/>
+ <entity name='gtreqless' codepoint='22DB'/>
+ <entity name='gtreqqless' codepoint='2A8C'/>
+ <entity name='gtrless' codepoint='2277'/>
+ <entity name='gtrsim' codepoint='2273'/>
+ <entity name='Hacek' codepoint='02C7'/>
+ <entity name='hairsp' codepoint='200A'/>
+ <entity name='half' codepoint='00BD'/>
+ <entity name='hamilt' codepoint='210B'/>
+ <entity name='HARDcy' codepoint='042A'/>
+ <entity name='hardcy' codepoint='044A'/>
+ <entity name='harr' codepoint='2194'/>
+ <entity name='hArr' codepoint='21D4'/>
+ <entity name='harrcir' codepoint='2948'/>
+ <entity name='harrw' codepoint='21AD'/>
+ <entity name='Hat' codepoint='005E'/>
+ <entity name='hbar' codepoint='210F'/>
+ <entity name='Hcirc' codepoint='0124'/>
+ <entity name='hcirc' codepoint='0125'/>
+ <entity name='hearts' codepoint='2665'/>
+ <entity name='heartsuit' codepoint='2665'/>
+ <entity name='hellip' codepoint='2026'/>
+ <entity name='hercon' codepoint='22B9'/>
+ <entity name='Hfr' codepoint='210C'/>
+ <entity name='hfr' codepoint='1D525'/>
+ <entity name='HilbertSpace' codepoint='210B'/>
+ <entity name='hksearow' codepoint='2925'/>
+ <entity name='hkswarow' codepoint='2926'/>
+ <entity name='hoarr' codepoint='21FF'/>
+ <entity name='homtht' codepoint='223B'/>
+ <entity name='hookleftarrow' codepoint='21A9'/>
+ <entity name='hookrightarrow' codepoint='21AA'/>
+ <entity name='Hopf' codepoint='210D'/>
+ <entity name='hopf' codepoint='1D559'/>
+ <entity name='horbar' codepoint='2015'/>
+ <entity name='HorizontalLine' codepoint='2500'/>
+ <entity name='Hscr' codepoint='210B'/>
+ <entity name='hscr' codepoint='1D4BD'/>
+ <entity name='hslash' codepoint='210F'/>
+ <entity name='Hstrok' codepoint='0126'/>
+ <entity name='hstrok' codepoint='0127'/>
+ <entity name='HumpDownHump' codepoint='224E'/>
+ <entity name='HumpEqual' codepoint='224F'/>
+ <entity name='hybull' codepoint='2043'/>
+ <entity name='hyphen' codepoint='2010'/>
+ <entity name='Iacgr' codepoint='038A'/>
+ <entity name='iacgr' codepoint='03AF'/>
+ <entity name='Iacute' codepoint='00CD'/>
+ <entity name='iacute' codepoint='00ED'/>
+ <entity name='ic' codepoint='2063'/>
+ <entity name='Icirc' codepoint='00CE'/>
+ <entity name='icirc' codepoint='00EE'/>
+ <entity name='Icy' codepoint='0418'/>
+ <entity name='icy' codepoint='0438'/>
+ <entity name='idiagr' codepoint='0390'/>
+ <entity name='Idigr' codepoint='03AA'/>
+ <entity name='idigr' codepoint='03CA'/>
+ <entity name='Idot' codepoint='0130'/>
+ <entity name='IEcy' codepoint='0415'/>
+ <entity name='iecy' codepoint='0435'/>
+ <entity name='iexcl' codepoint='00A1'/>
+ <entity name='iff' codepoint='21D4'/>
+ <entity name='Ifr' codepoint='2111'/>
+ <entity name='ifr' codepoint='1D526'/>
+ <entity name='Igr' codepoint='0399'/>
+ <entity name='igr' codepoint='03B9'/>
+ <entity name='Igrave' codepoint='00CC'/>
+ <entity name='igrave' codepoint='00EC'/>
+ <entity name='ii' codepoint='2148'/>
+ <entity name='iiiint' codepoint='2A0C'/>
+ <entity name='iiint' codepoint='222D'/>
+ <entity name='iinfin' codepoint='29DC'/>
+ <entity name='iiota' codepoint='2129'/>
+ <entity name='IJlig' codepoint='0132'/>
+ <entity name='ijlig' codepoint='0133'/>
+ <entity name='Im' codepoint='2111'/>
+ <entity name='Imacr' codepoint='012A'/>
+ <entity name='imacr' codepoint='012B'/>
+ <entity name='image' codepoint='2111'/>
+ <entity name='ImaginaryI' codepoint='2148'/>
+ <entity name='imagline' codepoint='2110'/>
+ <entity name='imagpart' codepoint='2111'/>
+ <entity name='imath' codepoint='0131'/>
+ <entity name='imof' codepoint='22B7'/>
+ <entity name='imped' codepoint='01B5'/>
+ <entity name='Implies' codepoint='21D2'/>
+ <entity name='in' codepoint='2208'/>
+ <entity name='incare' codepoint='2105'/>
+ <entity name='infin' codepoint='221E'/>
+ <entity name='infintie' codepoint='29DD'/>
+ <entity name='inodot' codepoint='0131'/>
+ <entity name='int' codepoint='222B'/>
+ <entity name='Int' codepoint='222C'/>
+ <entity name='intcal' codepoint='22BA'/>
+ <entity name='integers' codepoint='2124'/>
+ <entity name='Integral' codepoint='222B'/>
+ <entity name='intercal' codepoint='22BA'/>
+ <entity name='Intersection' codepoint='22C2'/>
+ <entity name='intlarhk' codepoint='2A17'/>
+ <entity name='intprod' codepoint='2A3C'/>
+ <entity name='InvisibleComma' codepoint='2063'/>
+ <entity name='InvisibleTimes' codepoint='2062'/>
+ <entity name='IOcy' codepoint='0401'/>
+ <entity name='iocy' codepoint='0451'/>
+ <entity name='Iogon' codepoint='012E'/>
+ <entity name='iogon' codepoint='012F'/>
+ <entity name='Iopf' codepoint='1D540'/>
+ <entity name='iopf' codepoint='1D55A'/>
+ <entity name='Iota' codepoint='0399'/>
+ <entity name='iota' codepoint='03B9'/>
+ <entity name='iprod' codepoint='2A3C'/>
+ <entity name='iquest' codepoint='00BF'/>
+ <entity name='Iscr' codepoint='2110'/>
+ <entity name='iscr' codepoint='1D4BE'/>
+ <entity name='isin' codepoint='2208'/>
+ <entity name='isindot' codepoint='22F5'/>
+ <entity name='isinE' codepoint='22F9'/>
+ <entity name='isins' codepoint='22F4'/>
+ <entity name='isinsv' codepoint='22F3'/>
+ <entity name='isinv' codepoint='2208'/>
+ <entity name='it' codepoint='2062'/>
+ <entity name='Itilde' codepoint='0128'/>
+ <entity name='itilde' codepoint='0129'/>
+ <entity name='Iukcy' codepoint='0406'/>
+ <entity name='iukcy' codepoint='0456'/>
+ <entity name='Iuml' codepoint='00CF'/>
+ <entity name='iuml' codepoint='00EF'/>
+ <entity name='Jcirc' codepoint='0134'/>
+ <entity name='jcirc' codepoint='0135'/>
+ <entity name='Jcy' codepoint='0419'/>
+ <entity name='jcy' codepoint='0439'/>
+ <entity name='Jfr' codepoint='1D50D'/>
+ <entity name='jfr' codepoint='1D527'/>
+ <entity name='jmath' codepoint='0237'/>
+ <entity name='Jopf' codepoint='1D541'/>
+ <entity name='jopf' codepoint='1D55B'/>
+ <entity name='Jscr' codepoint='1D4A5'/>
+ <entity name='jscr' codepoint='1D4BF'/>
+ <entity name='Jsercy' codepoint='0408'/>
+ <entity name='jsercy' codepoint='0458'/>
+ <entity name='Jukcy' codepoint='0404'/>
+ <entity name='jukcy' codepoint='0454'/>
+ <entity name='Kappa' codepoint='039A'/>
+ <entity name='kappa' codepoint='03BA'/>
+ <entity name='kappav' codepoint='03F0'/>
+ <entity name='Kcedil' codepoint='0136'/>
+ <entity name='kcedil' codepoint='0137'/>
+ <entity name='Kcy' codepoint='041A'/>
+ <entity name='kcy' codepoint='043A'/>
+ <entity name='Kfr' codepoint='1D50E'/>
+ <entity name='kfr' codepoint='1D528'/>
+ <entity name='Kgr' codepoint='039A'/>
+ <entity name='kgr' codepoint='03BA'/>
+ <entity name='kgreen' codepoint='0138'/>
+ <entity name='KHcy' codepoint='0425'/>
+ <entity name='khcy' codepoint='0445'/>
+ <entity name='KHgr' codepoint='03A7'/>
+ <entity name='khgr' codepoint='03C7'/>
+ <entity name='KJcy' codepoint='040C'/>
+ <entity name='kjcy' codepoint='045C'/>
+ <entity name='Kopf' codepoint='1D542'/>
+ <entity name='kopf' codepoint='1D55C'/>
+ <entity name='Kscr' codepoint='1D4A6'/>
+ <entity name='kscr' codepoint='1D4C0'/>
+ <entity name='lAarr' codepoint='21DA'/>
+ <entity name='Lacute' codepoint='0139'/>
+ <entity name='lacute' codepoint='013A'/>
+ <entity name='laemptyv' codepoint='29B4'/>
+ <entity name='lagran' codepoint='2112'/>
+ <entity name='Lambda' codepoint='039B'/>
+ <entity name='lambda' codepoint='03BB'/>
+ <entity name='lang' codepoint='2329'/>
+ <entity name='Lang' codepoint='27EA'/>
+ <entity name='langd' codepoint='2991'/>
+ <entity name='langle' codepoint='2329'/>
+ <entity name='lap' codepoint='2A85'/>
+ <entity name='Laplacetrf' codepoint='2112'/>
+ <entity name='laquo' codepoint='00AB'/>
+ <entity name='larr' codepoint='2190'/>
+ <entity name='Larr' codepoint='219E'/>
+ <entity name='lArr' codepoint='21D0'/>
+ <entity name='larrb' codepoint='21E4'/>
+ <entity name='larrbfs' codepoint='291F'/>
+ <entity name='larrfs' codepoint='291D'/>
+ <entity name='larrhk' codepoint='21A9'/>
+ <entity name='larrlp' codepoint='21AB'/>
+ <entity name='larrpl' codepoint='2939'/>
+ <entity name='larrsim' codepoint='2973'/>
+ <entity name='larrtl' codepoint='21A2'/>
+ <entity name='lat' codepoint='2AAB'/>
+ <entity name='latail' codepoint='2919'/>
+ <entity name='lAtail' codepoint='291B'/>
+ <entity name='late' codepoint='2AAD'/>
+ <entity name='lbarr' codepoint='290C'/>
+ <entity name='lBarr' codepoint='290E'/>
+ <entity name='lbbrk' codepoint='2997'/>
+ <entity name='lbrace' codepoint='007B'/>
+ <entity name='lbrack' codepoint='005B'/>
+ <entity name='lbrke' codepoint='298B'/>
+ <entity name='lbrksld' codepoint='298F'/>
+ <entity name='lbrkslu' codepoint='298D'/>
+ <entity name='Lcaron' codepoint='013D'/>
+ <entity name='lcaron' codepoint='013E'/>
+ <entity name='Lcedil' codepoint='013B'/>
+ <entity name='lcedil' codepoint='013C'/>
+ <entity name='lceil' codepoint='2308'/>
+ <entity name='lcub' codepoint='007B'/>
+ <entity name='Lcy' codepoint='041B'/>
+ <entity name='lcy' codepoint='043B'/>
+ <entity name='ldca' codepoint='2936'/>
+ <entity name='ldquo' codepoint='201C'/>
+ <entity name='ldquor' codepoint='201E'/>
+ <entity name='ldrdhar' codepoint='2967'/>
+ <entity name='ldrushar' codepoint='294B'/>
+ <entity name='ldsh' codepoint='21B2'/>
+ <entity name='le' codepoint='2264'/>
+ <entity name='lE' codepoint='2266'/>
+ <entity name='LeftAngleBracket' codepoint='2329'/>
+ <entity name='leftarrow' codepoint='2190'/>
+ <entity name='Leftarrow' codepoint='21D0'/>
+ <entity name='LeftArrowBar' codepoint='21E4'/>
+ <entity name='LeftArrowRightArrow' codepoint='21C6'/>
+ <entity name='leftarrowtail' codepoint='21A2'/>
+ <entity name='LeftCeiling' codepoint='2308'/>
+ <entity name='LeftDoubleBracket' codepoint='27E6'/>
+ <entity name='LeftDownTeeVector' codepoint='2961'/>
+ <entity name='LeftDownVector' codepoint='21C3'/>
+ <entity name='LeftDownVectorBar' codepoint='2959'/>
+ <entity name='LeftFloor' codepoint='230A'/>
+ <entity name='leftharpoondown' codepoint='21BD'/>
+ <entity name='leftharpoonup' codepoint='21BC'/>
+ <entity name='leftleftarrows' codepoint='21C7'/>
+ <entity name='leftrightarrow' codepoint='2194'/>
+ <entity name='Leftrightarrow' codepoint='21D4'/>
+ <entity name='leftrightarrows' codepoint='21C6'/>
+ <entity name='leftrightharpoons' codepoint='21CB'/>
+ <entity name='leftrightsquigarrow' codepoint='21AD'/>
+ <entity name='LeftRightVector' codepoint='294E'/>
+ <entity name='LeftTee' codepoint='22A3'/>
+ <entity name='LeftTeeArrow' codepoint='21A4'/>
+ <entity name='LeftTeeVector' codepoint='295A'/>
+ <entity name='leftthreetimes' codepoint='22CB'/>
+ <entity name='LeftTriangle' codepoint='22B2'/>
+ <entity name='LeftTriangleBar' codepoint='29CF'/>
+ <entity name='LeftTriangleEqual' codepoint='22B4'/>
+ <entity name='LeftUpDownVector' codepoint='2951'/>
+ <entity name='LeftUpTeeVector' codepoint='2960'/>
+ <entity name='LeftUpVector' codepoint='21BF'/>
+ <entity name='LeftUpVectorBar' codepoint='2958'/>
+ <entity name='LeftVector' codepoint='21BC'/>
+ <entity name='LeftVectorBar' codepoint='2952'/>
+ <entity name='leg' codepoint='22DA'/>
+ <entity name='lEg' codepoint='2A8B'/>
+ <entity name='leq' codepoint='2264'/>
+ <entity name='leqq' codepoint='2266'/>
+ <entity name='leqslant' codepoint='2A7D'/>
+ <entity name='les' codepoint='2A7D'/>
+ <entity name='lescc' codepoint='2AA8'/>
+ <entity name='lesdot' codepoint='2A7F'/>
+ <entity name='lesdoto' codepoint='2A81'/>
+ <entity name='lesdotor' codepoint='2A83'/>
+ <entity name='lesges' codepoint='2A93'/>
+ <entity name='lessapprox' codepoint='2A85'/>
+ <entity name='lessdot' codepoint='22D6'/>
+ <entity name='lesseqgtr' codepoint='22DA'/>
+ <entity name='lesseqqgtr' codepoint='2A8B'/>
+ <entity name='LessEqualGreater' codepoint='22DA'/>
+ <entity name='LessFullEqual' codepoint='2266'/>
+ <entity name='LessGreater' codepoint='2276'/>
+ <entity name='lessgtr' codepoint='2276'/>
+ <entity name='LessLess' codepoint='2AA1'/>
+ <entity name='lesssim' codepoint='2272'/>
+ <entity name='LessSlantEqual' codepoint='2A7D'/>
+ <entity name='LessTilde' codepoint='2272'/>
+ <entity name='lfisht' codepoint='297C'/>
+ <entity name='lfloor' codepoint='230A'/>
+ <entity name='Lfr' codepoint='1D50F'/>
+ <entity name='lfr' codepoint='1D529'/>
+ <entity name='lg' codepoint='2276'/>
+ <entity name='lgE' codepoint='2A91'/>
+ <entity name='Lgr' codepoint='039B'/>
+ <entity name='lgr' codepoint='03BB'/>
+ <entity name='lHar' codepoint='2962'/>
+ <entity name='lhard' codepoint='21BD'/>
+ <entity name='lharu' codepoint='21BC'/>
+ <entity name='lharul' codepoint='296A'/>
+ <entity name='lhblk' codepoint='2584'/>
+ <entity name='LJcy' codepoint='0409'/>
+ <entity name='ljcy' codepoint='0459'/>
+ <entity name='ll' codepoint='226A'/>
+ <entity name='Ll' codepoint='22D8'/>
+ <entity name='llarr' codepoint='21C7'/>
+ <entity name='llcorner' codepoint='231E'/>
+ <entity name='Lleftarrow' codepoint='21DA'/>
+ <entity name='llhard' codepoint='296B'/>
+ <entity name='lltri' codepoint='25FA'/>
+ <entity name='Lmidot' codepoint='013F'/>
+ <entity name='lmidot' codepoint='0140'/>
+ <entity name='lmoust' codepoint='23B0'/>
+ <entity name='lmoustache' codepoint='23B0'/>
+ <entity name='lnap' codepoint='2A89'/>
+ <entity name='lnapprox' codepoint='2A89'/>
+ <entity name='lnE' codepoint='2268'/>
+ <entity name='lne' codepoint='2A87'/>
+ <entity name='lneq' codepoint='2A87'/>
+ <entity name='lneqq' codepoint='2268'/>
+ <entity name='lnsim' codepoint='22E6'/>
+ <entity name='loang' codepoint='27EC'/>
+ <entity name='loarr' codepoint='21FD'/>
+ <entity name='lobrk' codepoint='27E6'/>
+ <entity name='longleftarrow' codepoint='27F5'/>
+ <entity name='Longleftarrow' codepoint='27F8'/>
+ <entity name='longleftrightarrow' codepoint='27F7'/>
+ <entity name='Longleftrightarrow' codepoint='27FA'/>
+ <entity name='longmapsto' codepoint='27FC'/>
+ <entity name='longrightarrow' codepoint='27F6'/>
+ <entity name='Longrightarrow' codepoint='27F9'/>
+ <entity name='looparrowleft' codepoint='21AB'/>
+ <entity name='looparrowright' codepoint='21AC'/>
+ <entity name='lopar' codepoint='2985'/>
+ <entity name='Lopf' codepoint='1D543'/>
+ <entity name='lopf' codepoint='1D55D'/>
+ <entity name='loplus' codepoint='2A2D'/>
+ <entity name='lotimes' codepoint='2A34'/>
+ <entity name='lowast' codepoint='2217'/>
+ <entity name='lowbar' codepoint='005F'/>
+ <entity name='LowerLeftArrow' codepoint='2199'/>
+ <entity name='LowerRightArrow' codepoint='2198'/>
+ <entity name='loz' codepoint='25CA'/>
+ <entity name='lozenge' codepoint='25CA'/>
+ <entity name='lozf' codepoint='29EB'/>
+ <entity name='lpar' codepoint='0028'/>
+ <entity name='lparlt' codepoint='2993'/>
+ <entity name='lrarr' codepoint='21C6'/>
+ <entity name='lrcorner' codepoint='231F'/>
+ <entity name='lrhar' codepoint='21CB'/>
+ <entity name='lrhard' codepoint='296D'/>
+ <entity name='lrm' codepoint='200E'/>
+ <entity name='lrtri' codepoint='22BF'/>
+ <entity name='lsaquo' codepoint='2039'/>
+ <entity name='Lscr' codepoint='2112'/>
+ <entity name='lscr' codepoint='1D4C1'/>
+ <entity name='lsh' codepoint='21B0'/>
+ <entity name='lsim' codepoint='2272'/>
+ <entity name='lsime' codepoint='2A8D'/>
+ <entity name='lsimg' codepoint='2A8F'/>
+ <entity name='lsqb' codepoint='005B'/>
+ <entity name='lsquo' codepoint='2018'/>
+ <entity name='lsquor' codepoint='201A'/>
+ <entity name='Lstrok' codepoint='0141'/>
+ <entity name='lstrok' codepoint='0142'/>
+ <entity name='lt' codepoint='003C'/>
+ <entity name='Lt' codepoint='226A'/>
+ <entity name='ltcc' codepoint='2AA6'/>
+ <entity name='ltcir' codepoint='2A79'/>
+ <entity name='ltdot' codepoint='22D6'/>
+ <entity name='lthree' codepoint='22CB'/>
+ <entity name='ltimes' codepoint='22C9'/>
+ <entity name='ltlarr' codepoint='2976'/>
+ <entity name='ltquest' codepoint='2A7B'/>
+ <entity name='ltri' codepoint='25C3'/>
+ <entity name='ltrie' codepoint='22B4'/>
+ <entity name='ltrif' codepoint='25C2'/>
+ <entity name='ltrPar' codepoint='2996'/>
+ <entity name='lurdshar' codepoint='294A'/>
+ <entity name='luruhar' codepoint='2966'/>
+ <entity name='macr' codepoint='00AF'/>
+ <entity name='male' codepoint='2642'/>
+ <entity name='malt' codepoint='2720'/>
+ <entity name='maltese' codepoint='2720'/>
+ <entity name='map' codepoint='21A6'/>
+ <entity name='Map' codepoint='2905'/>
+ <entity name='mapsto' codepoint='21A6'/>
+ <entity name='mapstodown' codepoint='21A7'/>
+ <entity name='mapstoleft' codepoint='21A4'/>
+ <entity name='mapstoup' codepoint='21A5'/>
+ <entity name='marker' codepoint='25AE'/>
+ <entity name='mcomma' codepoint='2A29'/>
+ <entity name='Mcy' codepoint='041C'/>
+ <entity name='mcy' codepoint='043C'/>
+ <entity name='mdash' codepoint='2014'/>
+ <entity name='mDDot' codepoint='223A'/>
+ <entity name='measuredangle' codepoint='2221'/>
+ <entity name='MediumSpace' codepoint='205F'/>
+ <entity name='Mellintrf' codepoint='2133'/>
+ <entity name='Mfr' codepoint='1D510'/>
+ <entity name='mfr' codepoint='1D52A'/>
+ <entity name='Mgr' codepoint='039C'/>
+ <entity name='mgr' codepoint='03BC'/>
+ <entity name='mho' codepoint='2127'/>
+ <entity name='micro' codepoint='00B5'/>
+ <entity name='mid' codepoint='2223'/>
+ <entity name='midast' codepoint='002A'/>
+ <entity name='midcir' codepoint='2AF0'/>
+ <entity name='middot' codepoint='00B7'/>
+ <entity name='minus' codepoint='2212'/>
+ <entity name='minusb' codepoint='229F'/>
+ <entity name='minusd' codepoint='2238'/>
+ <entity name='minusdu' codepoint='2A2A'/>
+ <entity name='MinusPlus' codepoint='2213'/>
+ <entity name='mlcp' codepoint='2ADB'/>
+ <entity name='mldr' codepoint='2026'/>
+ <entity name='mnplus' codepoint='2213'/>
+ <entity name='models' codepoint='22A7'/>
+ <entity name='Mopf' codepoint='1D544'/>
+ <entity name='mopf' codepoint='1D55E'/>
+ <entity name='mp' codepoint='2213'/>
+ <entity name='Mscr' codepoint='2133'/>
+ <entity name='mscr' codepoint='1D4C2'/>
+ <entity name='mstpos' codepoint='223E'/>
+ <entity name='Mu' codepoint='039C'/>
+ <entity name='mu' codepoint='03BC'/>
+ <entity name='multimap' codepoint='22B8'/>
+ <entity name='mumap' codepoint='22B8'/>
+ <entity name='nabla' codepoint='2207'/>
+ <entity name='Nacute' codepoint='0143'/>
+ <entity name='nacute' codepoint='0144'/>
+ <entity name='nap' codepoint='2249'/>
+ <entity name='napos' codepoint='0149'/>
+ <entity name='napprox' codepoint='2249'/>
+ <entity name='natur' codepoint='266E'/>
+ <entity name='natural' codepoint='266E'/>
+ <entity name='naturals' codepoint='2115'/>
+ <entity name='nbsp' codepoint='00A0'/>
+ <entity name='ncap' codepoint='2A43'/>
+ <entity name='Ncaron' codepoint='0147'/>
+ <entity name='ncaron' codepoint='0148'/>
+ <entity name='Ncedil' codepoint='0145'/>
+ <entity name='ncedil' codepoint='0146'/>
+ <entity name='ncong' codepoint='2247'/>
+ <entity name='ncup' codepoint='2A42'/>
+ <entity name='Ncy' codepoint='041D'/>
+ <entity name='ncy' codepoint='043D'/>
+ <entity name='ndash' codepoint='2013'/>
+ <entity name='ne' codepoint='2260'/>
+ <entity name='nearhk' codepoint='2924'/>
+ <entity name='nearr' codepoint='2197'/>
+ <entity name='neArr' codepoint='21D7'/>
+ <entity name='nearrow' codepoint='2197'/>
+ <entity name='NegativeMediumSpace' codepoint='200B'/>
+ <entity name='NegativeThickSpace' codepoint='200B'/>
+ <entity name='NegativeThinSpace' codepoint='200B'/>
+ <entity name='NegativeVeryThinSpace' codepoint='200B'/>
+ <entity name='nequiv' codepoint='2262'/>
+ <entity name='nesear' codepoint='2928'/>
+ <entity name='NestedGreaterGreater' codepoint='226B'/>
+ <entity name='NestedLessLess' codepoint='226A'/>
+ <entity name='NewLine' codepoint='000A'/>
+ <entity name='nexist' codepoint='2204'/>
+ <entity name='nexists' codepoint='2204'/>
+ <entity name='Nfr' codepoint='1D511'/>
+ <entity name='nfr' codepoint='1D52B'/>
+ <entity name='nge' codepoint='2271'/>
+ <entity name='ngeq' codepoint='2271'/>
+ <entity name='Ngr' codepoint='039D'/>
+ <entity name='ngr' codepoint='03BD'/>
+ <entity name='ngsim' codepoint='2275'/>
+ <entity name='ngt' codepoint='226F'/>
+ <entity name='ngtr' codepoint='226F'/>
+ <entity name='nharr' codepoint='21AE'/>
+ <entity name='nhArr' codepoint='21CE'/>
+ <entity name='nhpar' codepoint='2AF2'/>
+ <entity name='ni' codepoint='220B'/>
+ <entity name='nis' codepoint='22FC'/>
+ <entity name='nisd' codepoint='22FA'/>
+ <entity name='niv' codepoint='220B'/>
+ <entity name='NJcy' codepoint='040A'/>
+ <entity name='njcy' codepoint='045A'/>
+ <entity name='nlarr' codepoint='219A'/>
+ <entity name='nlArr' codepoint='21CD'/>
+ <entity name='nldr' codepoint='2025'/>
+ <entity name='nle' codepoint='2270'/>
+ <entity name='nleftarrow' codepoint='219A'/>
+ <entity name='nLeftarrow' codepoint='21CD'/>
+ <entity name='nleftrightarrow' codepoint='21AE'/>
+ <entity name='nLeftrightarrow' codepoint='21CE'/>
+ <entity name='nleq' codepoint='2270'/>
+ <entity name='nless' codepoint='226E'/>
+ <entity name='nlsim' codepoint='2274'/>
+ <entity name='nlt' codepoint='226E'/>
+ <entity name='nltri' codepoint='22EA'/>
+ <entity name='nltrie' codepoint='22EC'/>
+ <entity name='nmid' codepoint='2224'/>
+ <entity name='NoBreak' codepoint='2060'/>
+ <entity name='NonBreakingSpace' codepoint='00A0'/>
+ <entity name='Nopf' codepoint='2115'/>
+ <entity name='nopf' codepoint='1D55F'/>
+ <entity name='not' codepoint='00AC'/>
+ <entity name='Not' codepoint='2AEC'/>
+ <entity name='NotCongruent' codepoint='2262'/>
+ <entity name='NotCupCap' codepoint='226D'/>
+ <entity name='NotDoubleVerticalBar' codepoint='2226'/>
+ <entity name='NotElement' codepoint='2209'/>
+ <entity name='NotEqual' codepoint='2260'/>
+ <entity name='NotExists' codepoint='2204'/>
+ <entity name='NotGreater' codepoint='226F'/>
+ <entity name='NotGreaterEqual' codepoint='2271'/>
+ <entity name='NotGreaterLess' codepoint='2279'/>
+ <entity name='NotGreaterTilde' codepoint='2275'/>
+ <entity name='notin' codepoint='2209'/>
+ <entity name='notinva' codepoint='2209'/>
+ <entity name='notinvb' codepoint='22F7'/>
+ <entity name='notinvc' codepoint='22F6'/>
+ <entity name='NotLeftTriangle' codepoint='22EA'/>
+ <entity name='NotLeftTriangleEqual' codepoint='22EC'/>
+ <entity name='NotLess' codepoint='226E'/>
+ <entity name='NotLessEqual' codepoint='2270'/>
+ <entity name='NotLessGreater' codepoint='2278'/>
+ <entity name='NotLessTilde' codepoint='2274'/>
+ <entity name='notni' codepoint='220C'/>
+ <entity name='notniva' codepoint='220C'/>
+ <entity name='notnivb' codepoint='22FE'/>
+ <entity name='notnivc' codepoint='22FD'/>
+ <entity name='NotPrecedes' codepoint='2280'/>
+ <entity name='NotPrecedesSlantEqual' codepoint='22E0'/>
+ <entity name='NotReverseElement' codepoint='220C'/>
+ <entity name='NotRightTriangle' codepoint='22EB'/>
+ <entity name='NotRightTriangleEqual' codepoint='22ED'/>
+ <entity name='NotSquareSubsetEqual' codepoint='22E2'/>
+ <entity name='NotSquareSupersetEqual' codepoint='22E3'/>
+ <entity name='NotSubsetEqual' codepoint='2288'/>
+ <entity name='NotSucceeds' codepoint='2281'/>
+ <entity name='NotSucceedsSlantEqual' codepoint='22E1'/>
+ <entity name='NotSupersetEqual' codepoint='2289'/>
+ <entity name='NotTilde' codepoint='2241'/>
+ <entity name='NotTildeEqual' codepoint='2244'/>
+ <entity name='NotTildeFullEqual' codepoint='2247'/>
+ <entity name='NotTildeTilde' codepoint='2249'/>
+ <entity name='NotVerticalBar' codepoint='2224'/>
+ <entity name='npar' codepoint='2226'/>
+ <entity name='nparallel' codepoint='2226'/>
+ <entity name='npolint' codepoint='2A14'/>
+ <entity name='npr' codepoint='2280'/>
+ <entity name='nprcue' codepoint='22E0'/>
+ <entity name='nprec' codepoint='2280'/>
+ <entity name='nrarr' codepoint='219B'/>
+ <entity name='nrArr' codepoint='21CF'/>
+ <entity name='nrightarrow' codepoint='219B'/>
+ <entity name='nRightarrow' codepoint='21CF'/>
+ <entity name='nrtri' codepoint='22EB'/>
+ <entity name='nrtrie' codepoint='22ED'/>
+ <entity name='nsc' codepoint='2281'/>
+ <entity name='nsccue' codepoint='22E1'/>
+ <entity name='Nscr' codepoint='1D4A9'/>
+ <entity name='nscr' codepoint='1D4C3'/>
+ <entity name='nshortmid' codepoint='2224'/>
+ <entity name='nshortparallel' codepoint='2226'/>
+ <entity name='nsim' codepoint='2241'/>
+ <entity name='nsime' codepoint='2244'/>
+ <entity name='nsimeq' codepoint='2244'/>
+ <entity name='nsmid' codepoint='2224'/>
+ <entity name='nspar' codepoint='2226'/>
+ <entity name='nsqsube' codepoint='22E2'/>
+ <entity name='nsqsupe' codepoint='22E3'/>
+ <entity name='nsub' codepoint='2284'/>
+ <entity name='nsube' codepoint='2288'/>
+ <entity name='nsubseteq' codepoint='2288'/>
+ <entity name='nsucc' codepoint='2281'/>
+ <entity name='nsup' codepoint='2285'/>
+ <entity name='nsupe' codepoint='2289'/>
+ <entity name='nsupseteq' codepoint='2289'/>
+ <entity name='ntgl' codepoint='2279'/>
+ <entity name='Ntilde' codepoint='00D1'/>
+ <entity name='ntilde' codepoint='00F1'/>
+ <entity name='ntlg' codepoint='2278'/>
+ <entity name='ntriangleleft' codepoint='22EA'/>
+ <entity name='ntrianglelefteq' codepoint='22EC'/>
+ <entity name='ntriangleright' codepoint='22EB'/>
+ <entity name='ntrianglerighteq' codepoint='22ED'/>
+ <entity name='Nu' codepoint='039D'/>
+ <entity name='nu' codepoint='03BD'/>
+ <entity name='num' codepoint='0023'/>
+ <entity name='numero' codepoint='2116'/>
+ <entity name='numsp' codepoint='2007'/>
+ <entity name='nvdash' codepoint='22AC'/>
+ <entity name='nvDash' codepoint='22AD'/>
+ <entity name='nVdash' codepoint='22AE'/>
+ <entity name='nVDash' codepoint='22AF'/>
+ <entity name='nvHarr' codepoint='2904'/>
+ <entity name='nvinfin' codepoint='29DE'/>
+ <entity name='nvlArr' codepoint='2902'/>
+ <entity name='nvrArr' codepoint='2903'/>
+ <entity name='nwarhk' codepoint='2923'/>
+ <entity name='nwarr' codepoint='2196'/>
+ <entity name='nwArr' codepoint='21D6'/>
+ <entity name='nwarrow' codepoint='2196'/>
+ <entity name='nwnear' codepoint='2927'/>
+ <entity name='Oacgr' codepoint='038C'/>
+ <entity name='oacgr' codepoint='03CC'/>
+ <entity name='Oacute' codepoint='00D3'/>
+ <entity name='oacute' codepoint='00F3'/>
+ <entity name='oast' codepoint='229B'/>
+ <entity name='ocir' codepoint='229A'/>
+ <entity name='Ocirc' codepoint='00D4'/>
+ <entity name='ocirc' codepoint='00F4'/>
+ <entity name='Ocy' codepoint='041E'/>
+ <entity name='ocy' codepoint='043E'/>
+ <entity name='odash' codepoint='229D'/>
+ <entity name='Odblac' codepoint='0150'/>
+ <entity name='odblac' codepoint='0151'/>
+ <entity name='odiv' codepoint='2A38'/>
+ <entity name='odot' codepoint='2299'/>
+ <entity name='odsold' codepoint='29BC'/>
+ <entity name='OElig' codepoint='0152'/>
+ <entity name='oelig' codepoint='0153'/>
+ <entity name='ofcir' codepoint='29BF'/>
+ <entity name='Ofr' codepoint='1D512'/>
+ <entity name='ofr' codepoint='1D52C'/>
+ <entity name='ogon' codepoint='02DB'/>
+ <entity name='Ogr' codepoint='039F'/>
+ <entity name='ogr' codepoint='03BF'/>
+ <entity name='Ograve' codepoint='00D2'/>
+ <entity name='ograve' codepoint='00F2'/>
+ <entity name='ogt' codepoint='29C1'/>
+ <entity name='OHacgr' codepoint='038F'/>
+ <entity name='ohacgr' codepoint='03CE'/>
+ <entity name='ohbar' codepoint='29B5'/>
+ <entity name='OHgr' codepoint='03A9'/>
+ <entity name='ohgr' codepoint='03C9'/>
+ <entity name='ohm' codepoint='2126'/>
+ <entity name='oint' codepoint='222E'/>
+ <entity name='olarr' codepoint='21BA'/>
+ <entity name='olcir' codepoint='29BE'/>
+ <entity name='olcross' codepoint='29BB'/>
+ <entity name='oline' codepoint='203E'/>
+ <entity name='olt' codepoint='29C0'/>
+ <entity name='Omacr' codepoint='014C'/>
+ <entity name='omacr' codepoint='014D'/>
+ <entity name='Omega' codepoint='03A9'/>
+ <entity name='omega' codepoint='03C9'/>
+ <entity name='Omicron' codepoint='039F'/>
+ <entity name='omicron' codepoint='03BF'/>
+ <entity name='omid' codepoint='29B6'/>
+ <entity name='ominus' codepoint='2296'/>
+ <entity name='Oopf' codepoint='1D546'/>
+ <entity name='oopf' codepoint='1D560'/>
+ <entity name='opar' codepoint='29B7'/>
+ <entity name='OpenCurlyDoubleQuote' codepoint='201C'/>
+ <entity name='OpenCurlyQuote' codepoint='2018'/>
+ <entity name='operp' codepoint='29B9'/>
+ <entity name='oplus' codepoint='2295'/>
+ <entity name='or' codepoint='2228'/>
+ <entity name='Or' codepoint='2A54'/>
+ <entity name='orarr' codepoint='21BB'/>
+ <entity name='ord' codepoint='2A5D'/>
+ <entity name='order' codepoint='2134'/>
+ <entity name='orderof' codepoint='2134'/>
+ <entity name='ordf' codepoint='00AA'/>
+ <entity name='ordm' codepoint='00BA'/>
+ <entity name='origof' codepoint='22B6'/>
+ <entity name='oror' codepoint='2A56'/>
+ <entity name='orslope' codepoint='2A57'/>
+ <entity name='orv' codepoint='2A5B'/>
+ <entity name='oS' codepoint='24C8'/>
+ <entity name='oscr' codepoint='2134'/>
+ <entity name='Oscr' codepoint='1D4AA'/>
+ <entity name='Oslash' codepoint='00D8'/>
+ <entity name='oslash' codepoint='00F8'/>
+ <entity name='osol' codepoint='2298'/>
+ <entity name='Otilde' codepoint='00D5'/>
+ <entity name='otilde' codepoint='00F5'/>
+ <entity name='otimes' codepoint='2297'/>
+ <entity name='Otimes' codepoint='2A37'/>
+ <entity name='otimesas' codepoint='2A36'/>
+ <entity name='Ouml' codepoint='00D6'/>
+ <entity name='ouml' codepoint='00F6'/>
+ <entity name='ovbar' codepoint='233D'/>
+ <entity name='OverBar' codepoint='00AF'/>
+ <entity name='OverBrace' codepoint='FE37'/>
+ <entity name='OverBracket' codepoint='23B4'/>
+ <entity name='OverParenthesis' codepoint='FE35'/>
+ <entity name='par' codepoint='2225'/>
+ <entity name='para' codepoint='00B6'/>
+ <entity name='parallel' codepoint='2225'/>
+ <entity name='parsim' codepoint='2AF3'/>
+ <entity name='parsl' codepoint='2AFD'/>
+ <entity name='part' codepoint='2202'/>
+ <entity name='PartialD' codepoint='2202'/>
+ <entity name='Pcy' codepoint='041F'/>
+ <entity name='pcy' codepoint='043F'/>
+ <entity name='percnt' codepoint='0025'/>
+ <entity name='period' codepoint='002E'/>
+ <entity name='permil' codepoint='2030'/>
+ <entity name='perp' codepoint='22A5'/>
+ <entity name='pertenk' codepoint='2031'/>
+ <entity name='Pfr' codepoint='1D513'/>
+ <entity name='pfr' codepoint='1D52D'/>
+ <entity name='Pgr' codepoint='03A0'/>
+ <entity name='pgr' codepoint='03C0'/>
+ <entity name='PHgr' codepoint='03A6'/>
+ <entity name='phgr' codepoint='03C6'/>
+ <entity name='Phi' codepoint='03A6'/>
+ <entity name='phi' codepoint='03D5'/>
+ <entity name='phiv' codepoint='03C6'/>
+ <entity name='phmmat' codepoint='2133'/>
+ <entity name='phone' codepoint='260E'/>
+ <entity name='Pi' codepoint='03A0'/>
+ <entity name='pi' codepoint='03C0'/>
+ <entity name='pitchfork' codepoint='22D4'/>
+ <entity name='piv' codepoint='03D6'/>
+ <entity name='planck' codepoint='210F'/>
+ <entity name='planckh' codepoint='210E'/>
+ <entity name='plankv' codepoint='210F'/>
+ <entity name='plus' codepoint='002B'/>
+ <entity name='plusacir' codepoint='2A23'/>
+ <entity name='plusb' codepoint='229E'/>
+ <entity name='pluscir' codepoint='2A22'/>
+ <entity name='plusdo' codepoint='2214'/>
+ <entity name='plusdu' codepoint='2A25'/>
+ <entity name='pluse' codepoint='2A72'/>
+ <entity name='PlusMinus' codepoint='00B1'/>
+ <entity name='plusmn' codepoint='00B1'/>
+ <entity name='plussim' codepoint='2A26'/>
+ <entity name='plustwo' codepoint='2A27'/>
+ <entity name='pm' codepoint='00B1'/>
+ <entity name='Poincareplane' codepoint='210C'/>
+ <entity name='pointint' codepoint='2A15'/>
+ <entity name='Popf' codepoint='2119'/>
+ <entity name='popf' codepoint='1D561'/>
+ <entity name='pound' codepoint='00A3'/>
+ <entity name='pr' codepoint='227A'/>
+ <entity name='Pr' codepoint='2ABB'/>
+ <entity name='prap' codepoint='2AB7'/>
+ <entity name='prcue' codepoint='227C'/>
+ <entity name='pre' codepoint='2AAF'/>
+ <entity name='prE' codepoint='2AB3'/>
+ <entity name='prec' codepoint='227A'/>
+ <entity name='precapprox' codepoint='2AB7'/>
+ <entity name='preccurlyeq' codepoint='227C'/>
+ <entity name='Precedes' codepoint='227A'/>
+ <entity name='PrecedesEqual' codepoint='2AAF'/>
+ <entity name='PrecedesSlantEqual' codepoint='227C'/>
+ <entity name='PrecedesTilde' codepoint='227E'/>
+ <entity name='preceq' codepoint='2AAF'/>
+ <entity name='precnapprox' codepoint='2AB9'/>
+ <entity name='precneqq' codepoint='2AB5'/>
+ <entity name='precnsim' codepoint='22E8'/>
+ <entity name='precsim' codepoint='227E'/>
+ <entity name='prime' codepoint='2032'/>
+ <entity name='Prime' codepoint='2033'/>
+ <entity name='primes' codepoint='2119'/>
+ <entity name='prnap' codepoint='2AB9'/>
+ <entity name='prnE' codepoint='2AB5'/>
+ <entity name='prnsim' codepoint='22E8'/>
+ <entity name='prod' codepoint='220F'/>
+ <entity name='Product' codepoint='220F'/>
+ <entity name='profalar' codepoint='232E'/>
+ <entity name='profline' codepoint='2312'/>
+ <entity name='profsurf' codepoint='2313'/>
+ <entity name='prop' codepoint='221D'/>
+ <entity name='Proportion' codepoint='2237'/>
+ <entity name='Proportional' codepoint='221D'/>
+ <entity name='propto' codepoint='221D'/>
+ <entity name='prsim' codepoint='227E'/>
+ <entity name='prurel' codepoint='22B0'/>
+ <entity name='Pscr' codepoint='1D4AB'/>
+ <entity name='pscr' codepoint='1D4C5'/>
+ <entity name='PSgr' codepoint='03A8'/>
+ <entity name='psgr' codepoint='03C8'/>
+ <entity name='Psi' codepoint='03A8'/>
+ <entity name='psi' codepoint='03C8'/>
+ <entity name='puncsp' codepoint='2008'/>
+ <entity name='Qfr' codepoint='1D514'/>
+ <entity name='qfr' codepoint='1D52E'/>
+ <entity name='qint' codepoint='2A0C'/>
+ <entity name='Qopf' codepoint='211A'/>
+ <entity name='qopf' codepoint='1D562'/>
+ <entity name='qprime' codepoint='2057'/>
+ <entity name='Qscr' codepoint='1D4AC'/>
+ <entity name='qscr' codepoint='1D4C6'/>
+ <entity name='quaternions' codepoint='210D'/>
+ <entity name='quatint' codepoint='2A16'/>
+ <entity name='quest' codepoint='003F'/>
+ <entity name='questeq' codepoint='225F'/>
+ <entity name='quot' codepoint='0022'/>
+ <entity name='rAarr' codepoint='21DB'/>
+ <entity name='race' codepoint='29DA'/>
+ <entity name='Racute' codepoint='0154'/>
+ <entity name='racute' codepoint='0155'/>
+ <entity name='radic' codepoint='221A'/>
+ <entity name='raemptyv' codepoint='29B3'/>
+ <entity name='rang' codepoint='232A'/>
+ <entity name='Rang' codepoint='27EB'/>
+ <entity name='rangd' codepoint='2992'/>
+ <entity name='range' codepoint='29A5'/>
+ <entity name='rangle' codepoint='232A'/>
+ <entity name='raquo' codepoint='00BB'/>
+ <entity name='rarr' codepoint='2192'/>
+ <entity name='Rarr' codepoint='21A0'/>
+ <entity name='rArr' codepoint='21D2'/>
+ <entity name='rarrap' codepoint='2975'/>
+ <entity name='rarrb' codepoint='21E5'/>
+ <entity name='rarrbfs' codepoint='2920'/>
+ <entity name='rarrc' codepoint='2933'/>
+ <entity name='rarrfs' codepoint='291E'/>
+ <entity name='rarrhk' codepoint='21AA'/>
+ <entity name='rarrlp' codepoint='21AC'/>
+ <entity name='rarrpl' codepoint='2945'/>
+ <entity name='rarrsim' codepoint='2974'/>
+ <entity name='rarrtl' codepoint='21A3'/>
+ <entity name='Rarrtl' codepoint='2916'/>
+ <entity name='rarrw' codepoint='219D'/>
+ <entity name='ratail' codepoint='291A'/>
+ <entity name='rAtail' codepoint='291C'/>
+ <entity name='ratio' codepoint='2236'/>
+ <entity name='rationals' codepoint='211A'/>
+ <entity name='rbarr' codepoint='290D'/>
+ <entity name='rBarr' codepoint='290F'/>
+ <entity name='RBarr' codepoint='2910'/>
+ <entity name='rbbrk' codepoint='2998'/>
+ <entity name='rbrace' codepoint='007D'/>
+ <entity name='rbrack' codepoint='005D'/>
+ <entity name='rbrke' codepoint='298C'/>
+ <entity name='rbrksld' codepoint='298E'/>
+ <entity name='rbrkslu' codepoint='2990'/>
+ <entity name='Rcaron' codepoint='0158'/>
+ <entity name='rcaron' codepoint='0159'/>
+ <entity name='Rcedil' codepoint='0156'/>
+ <entity name='rcedil' codepoint='0157'/>
+ <entity name='rceil' codepoint='2309'/>
+ <entity name='rcub' codepoint='007D'/>
+ <entity name='Rcy' codepoint='0420'/>
+ <entity name='rcy' codepoint='0440'/>
+ <entity name='rdca' codepoint='2937'/>
+ <entity name='rdldhar' codepoint='2969'/>
+ <entity name='rdquo' codepoint='201D'/>
+ <entity name='rdquor' codepoint='201D'/>
+ <entity name='rdsh' codepoint='21B3'/>
+ <entity name='Re' codepoint='211C'/>
+ <entity name='real' codepoint='211C'/>
+ <entity name='realine' codepoint='211B'/>
+ <entity name='realpart' codepoint='211C'/>
+ <entity name='reals' codepoint='211D'/>
+ <entity name='rect' codepoint='25AD'/>
+ <entity name='reg' codepoint='00AE'/>
+ <entity name='ReverseElement' codepoint='220B'/>
+ <entity name='ReverseEquilibrium' codepoint='21CB'/>
+ <entity name='ReverseUpEquilibrium' codepoint='296F'/>
+ <entity name='rfisht' codepoint='297D'/>
+ <entity name='rfloor' codepoint='230B'/>
+ <entity name='Rfr' codepoint='211C'/>
+ <entity name='rfr' codepoint='1D52F'/>
+ <entity name='Rgr' codepoint='03A1'/>
+ <entity name='rgr' codepoint='03C1'/>
+ <entity name='rHar' codepoint='2964'/>
+ <entity name='rhard' codepoint='21C1'/>
+ <entity name='rharu' codepoint='21C0'/>
+ <entity name='rharul' codepoint='296C'/>
+ <entity name='Rho' codepoint='03A1'/>
+ <entity name='rho' codepoint='03C1'/>
+ <entity name='rhov' codepoint='03F1'/>
+ <entity name='RightAngleBracket' codepoint='232A'/>
+ <entity name='rightarrow' codepoint='2192'/>
+ <entity name='Rightarrow' codepoint='21D2'/>
+ <entity name='RightArrowBar' codepoint='21E5'/>
+ <entity name='RightArrowLeftArrow' codepoint='21C4'/>
+ <entity name='rightarrowtail' codepoint='21A3'/>
+ <entity name='RightCeiling' codepoint='2309'/>
+ <entity name='RightDoubleBracket' codepoint='27E7'/>
+ <entity name='RightDownTeeVector' codepoint='295D'/>
+ <entity name='RightDownVector' codepoint='21C2'/>
+ <entity name='RightDownVectorBar' codepoint='2955'/>
+ <entity name='RightFloor' codepoint='230B'/>
+ <entity name='rightharpoondown' codepoint='21C1'/>
+ <entity name='rightharpoonup' codepoint='21C0'/>
+ <entity name='rightleftarrows' codepoint='21C4'/>
+ <entity name='rightleftharpoons' codepoint='21CC'/>
+ <entity name='rightrightarrows' codepoint='21C9'/>
+ <entity name='rightsquigarrow' codepoint='219D'/>
+ <entity name='RightTee' codepoint='22A2'/>
+ <entity name='RightTeeArrow' codepoint='21A6'/>
+ <entity name='RightTeeVector' codepoint='295B'/>
+ <entity name='rightthreetimes' codepoint='22CC'/>
+ <entity name='RightTriangle' codepoint='22B3'/>
+ <entity name='RightTriangleBar' codepoint='29D0'/>
+ <entity name='RightTriangleEqual' codepoint='22B5'/>
+ <entity name='RightUpDownVector' codepoint='294F'/>
+ <entity name='RightUpTeeVector' codepoint='295C'/>
+ <entity name='RightUpVector' codepoint='21BE'/>
+ <entity name='RightUpVectorBar' codepoint='2954'/>
+ <entity name='RightVector' codepoint='21C0'/>
+ <entity name='RightVectorBar' codepoint='2953'/>
+ <entity name='ring' codepoint='02DA'/>
+ <entity name='risingdotseq' codepoint='2253'/>
+ <entity name='rlarr' codepoint='21C4'/>
+ <entity name='rlhar' codepoint='21CC'/>
+ <entity name='rlm' codepoint='200F'/>
+ <entity name='rmoust' codepoint='23B1'/>
+ <entity name='rmoustache' codepoint='23B1'/>
+ <entity name='rnmid' codepoint='2AEE'/>
+ <entity name='roang' codepoint='27ED'/>
+ <entity name='roarr' codepoint='21FE'/>
+ <entity name='robrk' codepoint='27E7'/>
+ <entity name='ropar' codepoint='2986'/>
+ <entity name='Ropf' codepoint='211D'/>
+ <entity name='ropf' codepoint='1D563'/>
+ <entity name='roplus' codepoint='2A2E'/>
+ <entity name='rotimes' codepoint='2A35'/>
+ <entity name='RoundImplies' codepoint='2970'/>
+ <entity name='rpar' codepoint='0029'/>
+ <entity name='rpargt' codepoint='2994'/>
+ <entity name='rppolint' codepoint='2A12'/>
+ <entity name='rrarr' codepoint='21C9'/>
+ <entity name='Rrightarrow' codepoint='21DB'/>
+ <entity name='rsaquo' codepoint='203A'/>
+ <entity name='Rscr' codepoint='211B'/>
+ <entity name='rscr' codepoint='1D4C7'/>
+ <entity name='rsh' codepoint='21B1'/>
+ <entity name='rsqb' codepoint='005D'/>
+ <entity name='rsquo' codepoint='2019'/>
+ <entity name='rsquor' codepoint='2019'/>
+ <entity name='rthree' codepoint='22CC'/>
+ <entity name='rtimes' codepoint='22CA'/>
+ <entity name='rtri' codepoint='25B9'/>
+ <entity name='rtrie' codepoint='22B5'/>
+ <entity name='rtrif' codepoint='25B8'/>
+ <entity name='rtriltri' codepoint='29CE'/>
+ <entity name='RuleDelayed' codepoint='29F4'/>
+ <entity name='ruluhar' codepoint='2968'/>
+ <entity name='rx' codepoint='211E'/>
+ <entity name='Sacute' codepoint='015A'/>
+ <entity name='sacute' codepoint='015B'/>
+ <entity name='sbquo' codepoint='201A'/>
+ <entity name='sc' codepoint='227B'/>
+ <entity name='Sc' codepoint='2ABC'/>
+ <entity name='scap' codepoint='2AB8'/>
+ <entity name='Scaron' codepoint='0160'/>
+ <entity name='scaron' codepoint='0161'/>
+ <entity name='sccue' codepoint='227D'/>
+ <entity name='sce' codepoint='2AB0'/>
+ <entity name='scE' codepoint='2AB4'/>
+ <entity name='Scedil' codepoint='015E'/>
+ <entity name='scedil' codepoint='015F'/>
+ <entity name='Scirc' codepoint='015C'/>
+ <entity name='scirc' codepoint='015D'/>
+ <entity name='scnap' codepoint='2ABA'/>
+ <entity name='scnE' codepoint='2AB6'/>
+ <entity name='scnsim' codepoint='22E9'/>
+ <entity name='scpolint' codepoint='2A13'/>
+ <entity name='scsim' codepoint='227F'/>
+ <entity name='Scy' codepoint='0421'/>
+ <entity name='scy' codepoint='0441'/>
+ <entity name='sdot' codepoint='22C5'/>
+ <entity name='sdotb' codepoint='22A1'/>
+ <entity name='sdote' codepoint='2A66'/>
+ <entity name='searhk' codepoint='2925'/>
+ <entity name='searr' codepoint='2198'/>
+ <entity name='seArr' codepoint='21D8'/>
+ <entity name='searrow' codepoint='2198'/>
+ <entity name='sect' codepoint='00A7'/>
+ <entity name='semi' codepoint='003B'/>
+ <entity name='seswar' codepoint='2929'/>
+ <entity name='setminus' codepoint='2216'/>
+ <entity name='setmn' codepoint='2216'/>
+ <entity name='sext' codepoint='2736'/>
+ <entity name='sfgr' codepoint='03C2'/>
+ <entity name='Sfr' codepoint='1D516'/>
+ <entity name='sfr' codepoint='1D530'/>
+ <entity name='sfrown' codepoint='2322'/>
+ <entity name='Sgr' codepoint='03A3'/>
+ <entity name='sgr' codepoint='03C3'/>
+ <entity name='sharp' codepoint='266F'/>
+ <entity name='SHCHcy' codepoint='0429'/>
+ <entity name='shchcy' codepoint='0449'/>
+ <entity name='SHcy' codepoint='0428'/>
+ <entity name='shcy' codepoint='0448'/>
+ <entity name='ShortDownArrow' codepoint='2193'/>
+ <entity name='ShortLeftArrow' codepoint='2190'/>
+ <entity name='shortmid' codepoint='2223'/>
+ <entity name='shortparallel' codepoint='2225'/>
+ <entity name='ShortRightArrow' codepoint='2192'/>
+ <entity name='ShortUpArrow' codepoint='2191'/>
+ <entity name='shy' codepoint='00AD'/>
+ <entity name='Sigma' codepoint='03A3'/>
+ <entity name='sigma' codepoint='03C3'/>
+ <entity name='sigmaf' codepoint='03C2'/>
+ <entity name='sigmav' codepoint='03C2'/>
+ <entity name='sim' codepoint='223C'/>
+ <entity name='simdot' codepoint='2A6A'/>
+ <entity name='sime' codepoint='2243'/>
+ <entity name='simeq' codepoint='2243'/>
+ <entity name='simg' codepoint='2A9E'/>
+ <entity name='simgE' codepoint='2AA0'/>
+ <entity name='siml' codepoint='2A9D'/>
+ <entity name='simlE' codepoint='2A9F'/>
+ <entity name='simne' codepoint='2246'/>
+ <entity name='simplus' codepoint='2A24'/>
+ <entity name='simrarr' codepoint='2972'/>
+ <entity name='slarr' codepoint='2190'/>
+ <entity name='SmallCircle' codepoint='2218'/>
+ <entity name='smallsetminus' codepoint='2216'/>
+ <entity name='smashp' codepoint='2A33'/>
+ <entity name='smeparsl' codepoint='29E4'/>
+ <entity name='smid' codepoint='2223'/>
+ <entity name='smile' codepoint='2323'/>
+ <entity name='smt' codepoint='2AAA'/>
+ <entity name='smte' codepoint='2AAC'/>
+ <entity name='SOFTcy' codepoint='042C'/>
+ <entity name='softcy' codepoint='044C'/>
+ <entity name='sol' codepoint='002F'/>
+ <entity name='solb' codepoint='29C4'/>
+ <entity name='solbar' codepoint='233F'/>
+ <entity name='Sopf' codepoint='1D54A'/>
+ <entity name='sopf' codepoint='1D564'/>
+ <entity name='spades' codepoint='2660'/>
+ <entity name='spadesuit' codepoint='2660'/>
+ <entity name='spar' codepoint='2225'/>
+ <entity name='sqcap' codepoint='2293'/>
+ <entity name='sqcup' codepoint='2294'/>
+ <entity name='Sqrt' codepoint='221A'/>
+ <entity name='sqsub' codepoint='228F'/>
+ <entity name='sqsube' codepoint='2291'/>
+ <entity name='sqsubset' codepoint='228F'/>
+ <entity name='sqsubseteq' codepoint='2291'/>
+ <entity name='sqsup' codepoint='2290'/>
+ <entity name='sqsupe' codepoint='2292'/>
+ <entity name='sqsupset' codepoint='2290'/>
+ <entity name='sqsupseteq' codepoint='2292'/>
+ <entity name='squ' codepoint='25A1'/>
+ <entity name='square' codepoint='25A1'/>
+ <entity name='SquareIntersection' codepoint='2293'/>
+ <entity name='SquareSubset' codepoint='228F'/>
+ <entity name='SquareSubsetEqual' codepoint='2291'/>
+ <entity name='SquareSuperset' codepoint='2290'/>
+ <entity name='SquareSupersetEqual' codepoint='2292'/>
+ <entity name='SquareUnion' codepoint='2294'/>
+ <entity name='squarf' codepoint='25AA'/>
+ <entity name='squf' codepoint='25AA'/>
+ <entity name='srarr' codepoint='2192'/>
+ <entity name='Sscr' codepoint='1D4AE'/>
+ <entity name='sscr' codepoint='1D4C8'/>
+ <entity name='ssetmn' codepoint='2216'/>
+ <entity name='ssmile' codepoint='2323'/>
+ <entity name='sstarf' codepoint='22C6'/>
+ <entity name='Star' codepoint='22C6'/>
+ <entity name='star' codepoint='2606'/>
+ <entity name='starf' codepoint='2605'/>
+ <entity name='straightepsilon' codepoint='03F5'/>
+ <entity name='straightphi' codepoint='03D5'/>
+ <entity name='strns' codepoint='00AF'/>
+ <entity name='sub' codepoint='2282'/>
+ <entity name='Sub' codepoint='22D0'/>
+ <entity name='subdot' codepoint='2ABD'/>
+ <entity name='sube' codepoint='2286'/>
+ <entity name='subE' codepoint='2AC5'/>
+ <entity name='subedot' codepoint='2AC3'/>
+ <entity name='submult' codepoint='2AC1'/>
+ <entity name='subne' codepoint='228A'/>
+ <entity name='subnE' codepoint='2ACB'/>
+ <entity name='subplus' codepoint='2ABF'/>
+ <entity name='subrarr' codepoint='2979'/>
+ <entity name='subset' codepoint='2282'/>
+ <entity name='Subset' codepoint='22D0'/>
+ <entity name='subseteq' codepoint='2286'/>
+ <entity name='subseteqq' codepoint='2AC5'/>
+ <entity name='SubsetEqual' codepoint='2286'/>
+ <entity name='subsetneq' codepoint='228A'/>
+ <entity name='subsetneqq' codepoint='2ACB'/>
+ <entity name='subsim' codepoint='2AC7'/>
+ <entity name='subsub' codepoint='2AD5'/>
+ <entity name='subsup' codepoint='2AD3'/>
+ <entity name='succ' codepoint='227B'/>
+ <entity name='succapprox' codepoint='2AB8'/>
+ <entity name='succcurlyeq' codepoint='227D'/>
+ <entity name='Succeeds' codepoint='227B'/>
+ <entity name='SucceedsEqual' codepoint='2AB0'/>
+ <entity name='SucceedsSlantEqual' codepoint='227D'/>
+ <entity name='SucceedsTilde' codepoint='227F'/>
+ <entity name='succeq' codepoint='2AB0'/>
+ <entity name='succnapprox' codepoint='2ABA'/>
+ <entity name='succneqq' codepoint='2AB6'/>
+ <entity name='succnsim' codepoint='22E9'/>
+ <entity name='succsim' codepoint='227F'/>
+ <entity name='SuchThat' codepoint='220B'/>
+ <entity name='sum' codepoint='2211'/>
+ <entity name='sung' codepoint='266A'/>
+ <entity name='sup' codepoint='2283'/>
+ <entity name='Sup' codepoint='22D1'/>
+ <entity name='sup1' codepoint='00B9'/>
+ <entity name='sup2' codepoint='00B2'/>
+ <entity name='sup3' codepoint='00B3'/>
+ <entity name='supdot' codepoint='2ABE'/>
+ <entity name='supdsub' codepoint='2AD8'/>
+ <entity name='supe' codepoint='2287'/>
+ <entity name='supE' codepoint='2AC6'/>
+ <entity name='supedot' codepoint='2AC4'/>
+ <entity name='Superset' codepoint='2283'/>
+ <entity name='SupersetEqual' codepoint='2287'/>
+ <entity name='suphsub' codepoint='2AD7'/>
+ <entity name='suplarr' codepoint='297B'/>
+ <entity name='supmult' codepoint='2AC2'/>
+ <entity name='supne' codepoint='228B'/>
+ <entity name='supnE' codepoint='2ACC'/>
+ <entity name='supplus' codepoint='2AC0'/>
+ <entity name='supset' codepoint='2283'/>
+ <entity name='Supset' codepoint='22D1'/>
+ <entity name='supseteq' codepoint='2287'/>
+ <entity name='supseteqq' codepoint='2AC6'/>
+ <entity name='supsetneq' codepoint='228B'/>
+ <entity name='supsetneqq' codepoint='2ACC'/>
+ <entity name='supsim' codepoint='2AC8'/>
+ <entity name='supsub' codepoint='2AD4'/>
+ <entity name='supsup' codepoint='2AD6'/>
+ <entity name='swarhk' codepoint='2926'/>
+ <entity name='swarr' codepoint='2199'/>
+ <entity name='swArr' codepoint='21D9'/>
+ <entity name='swarrow' codepoint='2199'/>
+ <entity name='swnwar' codepoint='292A'/>
+ <entity name='szlig' codepoint='00DF'/>
+ <entity name='Tab' codepoint='0009'/>
+ <entity name='target' codepoint='2316'/>
+ <entity name='Tau' codepoint='03A4'/>
+ <entity name='tau' codepoint='03C4'/>
+ <entity name='tbrk' codepoint='23B4'/>
+ <entity name='Tcaron' codepoint='0164'/>
+ <entity name='tcaron' codepoint='0165'/>
+ <entity name='Tcedil' codepoint='0162'/>
+ <entity name='tcedil' codepoint='0163'/>
+ <entity name='Tcy' codepoint='0422'/>
+ <entity name='tcy' codepoint='0442'/>
+ <entity name='telrec' codepoint='2315'/>
+ <entity name='Tfr' codepoint='1D517'/>
+ <entity name='tfr' codepoint='1D531'/>
+ <entity name='Tgr' codepoint='03A4'/>
+ <entity name='tgr' codepoint='03C4'/>
+ <entity name='there4' codepoint='2234'/>
+ <entity name='therefore' codepoint='2234'/>
+ <entity name='Theta' codepoint='0398'/>
+ <entity name='theta' codepoint='03B8'/>
+ <entity name='thetasym' codepoint='03D1'/>
+ <entity name='thetav' codepoint='03D1'/>
+ <entity name='THgr' codepoint='0398'/>
+ <entity name='thgr' codepoint='03B8'/>
+ <entity name='thickapprox' codepoint='2248'/>
+ <entity name='thicksim' codepoint='223C'/>
+ <entity name='thinsp' codepoint='2009'/>
+ <entity name='ThinSpace' codepoint='2009'/>
+ <entity name='thkap' codepoint='2248'/>
+ <entity name='thksim' codepoint='223C'/>
+ <entity name='THORN' codepoint='00DE'/>
+ <entity name='thorn' codepoint='00FE'/>
+ <entity name='tilde' codepoint='02DC'/>
+ <entity name='Tilde' codepoint='223C'/>
+ <entity name='TildeEqual' codepoint='2243'/>
+ <entity name='TildeFullEqual' codepoint='2245'/>
+ <entity name='TildeTilde' codepoint='2248'/>
+ <entity name='times' codepoint='00D7'/>
+ <entity name='timesb' codepoint='22A0'/>
+ <entity name='timesbar' codepoint='2A31'/>
+ <entity name='timesd' codepoint='2A30'/>
+ <entity name='tint' codepoint='222D'/>
+ <entity name='toea' codepoint='2928'/>
+ <entity name='top' codepoint='22A4'/>
+ <entity name='topbot' codepoint='2336'/>
+ <entity name='topcir' codepoint='2AF1'/>
+ <entity name='Topf' codepoint='1D54B'/>
+ <entity name='topf' codepoint='1D565'/>
+ <entity name='topfork' codepoint='2ADA'/>
+ <entity name='tosa' codepoint='2929'/>
+ <entity name='tprime' codepoint='2034'/>
+ <entity name='trade' codepoint='2122'/>
+ <entity name='triangle' codepoint='25B5'/>
+ <entity name='triangledown' codepoint='25BF'/>
+ <entity name='triangleleft' codepoint='25C3'/>
+ <entity name='trianglelefteq' codepoint='22B4'/>
+ <entity name='triangleq' codepoint='225C'/>
+ <entity name='triangleright' codepoint='25B9'/>
+ <entity name='trianglerighteq' codepoint='22B5'/>
+ <entity name='tridot' codepoint='25EC'/>
+ <entity name='trie' codepoint='225C'/>
+ <entity name='triminus' codepoint='2A3A'/>
+ <entity name='triplus' codepoint='2A39'/>
+ <entity name='trisb' codepoint='29CD'/>
+ <entity name='tritime' codepoint='2A3B'/>
+ <entity name='trpezium' codepoint='23E2'/>
+ <entity name='Tscr' codepoint='1D4AF'/>
+ <entity name='tscr' codepoint='1D4C9'/>
+ <entity name='TScy' codepoint='0426'/>
+ <entity name='tscy' codepoint='0446'/>
+ <entity name='TSHcy' codepoint='040B'/>
+ <entity name='tshcy' codepoint='045B'/>
+ <entity name='Tstrok' codepoint='0166'/>
+ <entity name='tstrok' codepoint='0167'/>
+ <entity name='twixt' codepoint='226C'/>
+ <entity name='twoheadleftarrow' codepoint='219E'/>
+ <entity name='twoheadrightarrow' codepoint='21A0'/>
+ <entity name='Uacgr' codepoint='038E'/>
+ <entity name='uacgr' codepoint='03CD'/>
+ <entity name='Uacute' codepoint='00DA'/>
+ <entity name='uacute' codepoint='00FA'/>
+ <entity name='uarr' codepoint='2191'/>
+ <entity name='Uarr' codepoint='219F'/>
+ <entity name='uArr' codepoint='21D1'/>
+ <entity name='Uarrocir' codepoint='2949'/>
+ <entity name='Ubrcy' codepoint='040E'/>
+ <entity name='ubrcy' codepoint='045E'/>
+ <entity name='Ubreve' codepoint='016C'/>
+ <entity name='ubreve' codepoint='016D'/>
+ <entity name='Ucirc' codepoint='00DB'/>
+ <entity name='ucirc' codepoint='00FB'/>
+ <entity name='Ucy' codepoint='0423'/>
+ <entity name='ucy' codepoint='0443'/>
+ <entity name='udarr' codepoint='21C5'/>
+ <entity name='Udblac' codepoint='0170'/>
+ <entity name='udblac' codepoint='0171'/>
+ <entity name='udhar' codepoint='296E'/>
+ <entity name='udiagr' codepoint='03B0'/>
+ <entity name='Udigr' codepoint='03AB'/>
+ <entity name='udigr' codepoint='03CB'/>
+ <entity name='ufisht' codepoint='297E'/>
+ <entity name='Ufr' codepoint='1D518'/>
+ <entity name='ufr' codepoint='1D532'/>
+ <entity name='Ugr' codepoint='03A5'/>
+ <entity name='ugr' codepoint='03C5'/>
+ <entity name='Ugrave' codepoint='00D9'/>
+ <entity name='ugrave' codepoint='00F9'/>
+ <entity name='uHar' codepoint='2963'/>
+ <entity name='uharl' codepoint='21BF'/>
+ <entity name='uharr' codepoint='21BE'/>
+ <entity name='uhblk' codepoint='2580'/>
+ <entity name='ulcorn' codepoint='231C'/>
+ <entity name='ulcorner' codepoint='231C'/>
+ <entity name='ulcrop' codepoint='230F'/>
+ <entity name='ultri' codepoint='25F8'/>
+ <entity name='Umacr' codepoint='016A'/>
+ <entity name='umacr' codepoint='016B'/>
+ <entity name='uml' codepoint='00A8'/>
+ <entity name='UnderBrace' codepoint='FE38'/>
+ <entity name='UnderBracket' codepoint='23B5'/>
+ <entity name='UnderParenthesis' codepoint='FE36'/>
+ <entity name='Union' codepoint='22C3'/>
+ <entity name='UnionPlus' codepoint='228E'/>
+ <entity name='Uogon' codepoint='0172'/>
+ <entity name='uogon' codepoint='0173'/>
+ <entity name='Uopf' codepoint='1D54C'/>
+ <entity name='uopf' codepoint='1D566'/>
+ <entity name='uparrow' codepoint='2191'/>
+ <entity name='Uparrow' codepoint='21D1'/>
+ <entity name='UpArrowBar' codepoint='2912'/>
+ <entity name='UpArrowDownArrow' codepoint='21C5'/>
+ <entity name='updownarrow' codepoint='2195'/>
+ <entity name='Updownarrow' codepoint='21D5'/>
+ <entity name='UpEquilibrium' codepoint='296E'/>
+ <entity name='upharpoonleft' codepoint='21BF'/>
+ <entity name='upharpoonright' codepoint='21BE'/>
+ <entity name='uplus' codepoint='228E'/>
+ <entity name='UpperLeftArrow' codepoint='2196'/>
+ <entity name='UpperRightArrow' codepoint='2197'/>
+ <entity name='upsi' codepoint='03C5'/>
+ <entity name='Upsi' codepoint='03D2'/>
+ <entity name='upsih' codepoint='03D2'/>
+ <entity name='Upsilon' codepoint='03A5'/>
+ <entity name='upsilon' codepoint='03C5'/>
+ <entity name='UpTee' codepoint='22A5'/>
+ <entity name='UpTeeArrow' codepoint='21A5'/>
+ <entity name='upuparrows' codepoint='21C8'/>
+ <entity name='urcorn' codepoint='231D'/>
+ <entity name='urcorner' codepoint='231D'/>
+ <entity name='urcrop' codepoint='230E'/>
+ <entity name='Uring' codepoint='016E'/>
+ <entity name='uring' codepoint='016F'/>
+ <entity name='urtri' codepoint='25F9'/>
+ <entity name='Uscr' codepoint='1D4B0'/>
+ <entity name='uscr' codepoint='1D4CA'/>
+ <entity name='utdot' codepoint='22F0'/>
+ <entity name='Utilde' codepoint='0168'/>
+ <entity name='utilde' codepoint='0169'/>
+ <entity name='utri' codepoint='25B5'/>
+ <entity name='utrif' codepoint='25B4'/>
+ <entity name='uuarr' codepoint='21C8'/>
+ <entity name='Uuml' codepoint='00DC'/>
+ <entity name='uuml' codepoint='00FC'/>
+ <entity name='uwangle' codepoint='29A7'/>
+ <entity name='vangrt' codepoint='299C'/>
+ <entity name='varepsilon' codepoint='03B5'/>
+ <entity name='varkappa' codepoint='03F0'/>
+ <entity name='varnothing' codepoint='2205'/>
+ <entity name='varphi' codepoint='03C6'/>
+ <entity name='varpi' codepoint='03D6'/>
+ <entity name='varpropto' codepoint='221D'/>
+ <entity name=
<TRUNCATED>
[15/33] lucenenet git commit: Corrected bug in Lifetime reference
count
Posted by ni...@apache.org.
Corrected bug in Lifetime reference count
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/6cf6a6e5
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/6cf6a6e5
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/6cf6a6e5
Branch: refs/heads/master
Commit: 6cf6a6e51453c74c6b4f26a8bd2d2be881cb4888
Parents: 1cfbd8b
Author: Vincent Van Den Berghe <vv...@bvdep.com>
Authored: Sun Jul 30 18:40:47 2017 +0200
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 19:33:25 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net/Search/SearcherLifetimeManager.cs | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/6cf6a6e5/src/Lucene.Net/Search/SearcherLifetimeManager.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Search/SearcherLifetimeManager.cs b/src/Lucene.Net/Search/SearcherLifetimeManager.cs
index 8b7b3d4..7c51a58 100644
--- a/src/Lucene.Net/Search/SearcherLifetimeManager.cs
+++ b/src/Lucene.Net/Search/SearcherLifetimeManager.cs
@@ -145,7 +145,7 @@ namespace Lucene.Net.Search
// TODO: we could get by w/ just a "set"; need to have
// Tracker hash by its version and have compareTo(Long)
// compare to its version
- private readonly ConcurrentDictionary<long, SearcherTracker> _searchers = new ConcurrentDictionary<long, SearcherTracker>();
+ private readonly ConcurrentDictionary<long, Lazy<SearcherTracker>> _searchers = new ConcurrentDictionary<long, Lazy<SearcherTracker>>();
private void EnsureOpen()
{
@@ -177,7 +177,7 @@ namespace Lucene.Net.Search
// bug isolation if we assign our own private ID:
var version = ((DirectoryReader)searcher.IndexReader).Version;
var factoryMethodCalled = false;
- var tracker = _searchers.GetOrAdd(version, l => { factoryMethodCalled = true; return new SearcherTracker(searcher); });
+ var tracker = _searchers.GetOrAdd(version, l => new Lazy<SearcherTracker>(() => { factoryMethodCalled = true; return new SearcherTracker(searcher); })).Value;
if (!factoryMethodCalled && tracker.Searcher != searcher)
{
throw new ArgumentException("the provided searcher has the same underlying reader version yet the searcher instance differs from before (new=" + searcher + " vs old=" + tracker.Searcher);
@@ -203,10 +203,10 @@ namespace Lucene.Net.Search
public virtual IndexSearcher Acquire(long version)
{
EnsureOpen();
- SearcherTracker tracker;
- if (_searchers.TryGetValue(version, out tracker) && tracker != null && tracker.Searcher.IndexReader.TryIncRef())
+ Lazy<SearcherTracker> tracker;
+ if (_searchers.TryGetValue(version, out tracker) && tracker.IsValueCreated && tracker.Value.Searcher.IndexReader.TryIncRef())
{
- return tracker.Searcher;
+ return tracker.Value.Searcher;
}
return null;
@@ -279,7 +279,7 @@ namespace Lucene.Net.Search
// (not thread-safe since the values can change while
// ArrayList is init'ing itself); must instead iterate
// ourselves:
- var trackers = _searchers.Values.ToList();
+ var trackers = _searchers.Values.Select(item => item.Value).ToList();
trackers.Sort();
var lastRecordTimeSec = 0.0;
double now = Time.NanoTime() / NANOS_PER_SEC;
@@ -301,7 +301,7 @@ namespace Lucene.Net.Search
if (pruner.DoPrune(ageSec, tracker.Searcher))
{
//System.out.println("PRUNE version=" + tracker.version + " age=" + ageSec + " ms=" + System.currentTimeMillis());
- SearcherTracker _;
+ Lazy<SearcherTracker> _;
_searchers.TryRemove(tracker.Version, out _);
tracker.Dispose();
}
@@ -326,13 +326,13 @@ namespace Lucene.Net.Search
lock (this)
{
_closed = true;
- IList<SearcherTracker> toClose = new List<SearcherTracker>(_searchers.Values);
+ IList<SearcherTracker> toClose = new List<SearcherTracker>(_searchers.Values.Select(item => item.Value));
// Remove up front in case exc below, so we don't
// over-decRef on double-close:
foreach (var tracker in toClose)
{
- SearcherTracker _;
+ Lazy<SearcherTracker> _;
_searchers.TryRemove(tracker.Version, out _);
}
[33/33] lucenenet git commit: Reverted back to icu.net-54.1.1-alpha,
since the icu-dotnet-2.2.0-netcore0078 package has some ICU loading
problems.
Posted by ni...@apache.org.
Reverted back to icu.net-54.1.1-alpha, since the icu-dotnet-2.2.0-netcore0078 package has some ICU loading problems.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/4782065c
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/4782065c
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/4782065c
Branch: refs/heads/master
Commit: 4782065ca7fc5a129c1e7820956df2fb68472441
Parents: 4a8507a
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Mon Aug 7 00:55:35 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Mon Aug 7 00:55:35 2017 +0700
----------------------------------------------------------------------
NuGet.config | 2 +-
.../Lucene.Net.Analysis.SmartCn.project.json | 2 +-
src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json | 2 +-
src/Lucene.Net.ICU/Lucene.Net.ICU.project.json | 2 +-
src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json | 2 +-
.../Lucene.Net.Tests.Benchmark.project.json | 2 +-
src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/NuGet.config
----------------------------------------------------------------------
diff --git a/NuGet.config b/NuGet.config
index 021fe6f..64c38ad 100644
--- a/NuGet.config
+++ b/NuGet.config
@@ -22,7 +22,7 @@
<configuration>
<packageSources>
<clear />
- <add key="icu-dotnet" value="https://www.myget.org/F/lucene-icu-dotnet/api/v2" />
+ <add key="icunet" value="https://www.myget.org/F/icu-dotnet/api/v2" />
<add key="dotnet-cat" value="https://www.myget.org/F/dotnetcat/api/v2" />
<add key="nugetorg" value="https://www.nuget.org/api/v2" />
</packageSources>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json b/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
index 1551a49..74d9d80 100644
--- a/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
+++ b/src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu-dotnet": "2.2.0-netcore0078"
+ "icu.net": "54.1.1-alpha"
},
"frameworks": {
"net451": {}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
index 83d487f..f764f6a 100644
--- a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu-dotnet": "2.2.0-netcore0078",
+ "icu.net": "54.1.1-alpha",
"SharpZipLib": "0.86.0",
"Spatial4n.Core": "0.4.1-beta00003"
},
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json b/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
index 496bea5..af28fc8 100644
--- a/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
+++ b/src/Lucene.Net.ICU/Lucene.Net.ICU.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu-dotnet": "2.2.0-netcore0078"
+ "icu.net": "54.1.1-alpha"
},
"frameworks": {
"net451": {}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json b/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
index b217af9..36faa44 100644
--- a/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
+++ b/src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu-dotnet": "2.2.0-netcore0078",
+ "icu.net": "54.1.1-alpha",
"NUnit": "3.5.0"
},
"frameworks": {
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
index ec41dd3..41388a7 100644
--- a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
+++ b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.project.json
@@ -3,7 +3,7 @@
"win": {}
},
"dependencies": {
- "icu-dotnet": "2.2.0-netcore0078",
+ "icu.net": "54.1.1-alpha",
"NUnit": "3.5.0",
"SharpZipLib": "0.86.0"
},
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4782065c/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json b/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
index fd1d44f..2a96d5b 100644
--- a/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
+++ b/src/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.project.json
@@ -4,7 +4,7 @@
},
"dependencies": {
"NUnit": "3.5.0",
- "icu-dotnet": "2.2.0-netcore0078"
+ "icu.net": "54.1.1-alpha"
},
"frameworks": {
"net451": {}
[07/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ReopenReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ReopenReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReopenReaderTask.cs
new file mode 100644
index 0000000..eeb8f8c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReopenReaderTask.cs
@@ -0,0 +1,45 @@
+using Lucene.Net.Index;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Reopens <see cref="IndexReader"/> and closes old <see cref="IndexReader"/>.
+ /// </summary>
+ public class ReopenReaderTask : PerfTask
+ {
+ public ReopenReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ DirectoryReader r = RunData.GetIndexReader();
+ DirectoryReader nr = DirectoryReader.OpenIfChanged(r);
+ if (nr != null)
+ {
+ RunData.SetIndexReader(nr);
+ nr.DecRef();
+ }
+ r.DecRef();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RepAllTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RepAllTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepAllTask.cs
new file mode 100644
index 0000000..e382605
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepAllTask.cs
@@ -0,0 +1,83 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report all statistics with no aggregations.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class RepAllTask : ReportTask
+ {
+ public RepAllTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ Report rp = ReportAll(RunData.Points.TaskStats);
+
+ SystemConsole.WriteLine();
+ SystemConsole.WriteLine("------------> Report All (" + rp.Count + " out of " + rp.OutOf + ")");
+ SystemConsole.WriteLine(rp.Text);
+ SystemConsole.WriteLine();
+ return 0;
+ }
+
+ /// <summary>
+ /// Report detailed statistics as a string.
+ /// </summary>
+ /// <param name="taskStats"></param>
+ /// <returns>The report.</returns>
+ protected virtual Report ReportAll(IList<TaskStats> taskStats)
+ {
+ string longestOp = LongestOp(taskStats);
+ bool first = true;
+ StringBuilder sb = new StringBuilder();
+ sb.Append(TableTitle(longestOp));
+ sb.Append(newline);
+ int reported = 0;
+ foreach (TaskStats stat in taskStats)
+ {
+ if (stat.Elapsed >= 0)
+ { // consider only tasks that ended
+ if (!first)
+ {
+ sb.Append(newline);
+ }
+ first = false;
+ string line = TaskReportLine(longestOp, stat);
+ reported++;
+ if (taskStats.Count > 2 && reported % 2 == 0)
+ {
+ line = line.Replace(" ", " - ");
+ }
+ sb.Append(line);
+ }
+ }
+ string reptxt = (reported == 0 ? "No Matching Entries Were Found!" : sb.ToString());
+ return new Report(reptxt, reported, reported, taskStats.Count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSelectByPrefTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSelectByPrefTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSelectByPrefTask.cs
new file mode 100644
index 0000000..16aac93
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSelectByPrefTask.cs
@@ -0,0 +1,81 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report by-name-prefix statistics with no aggregations.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class RepSelectByPrefTask : RepSumByPrefTask
+ {
+ public RepSelectByPrefTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ Report rp = ReportSelectByPrefix(RunData.Points.TaskStats);
+
+ SystemConsole.WriteLine();
+ SystemConsole.WriteLine("------------> Report Select By Prefix (" + m_prefix + ") (" +
+ rp.Count + " about " + rp.Reported + " out of " + rp.OutOf + ")");
+ SystemConsole.WriteLine(rp.Text);
+ SystemConsole.WriteLine();
+
+ return 0;
+ }
+
+ protected virtual Report ReportSelectByPrefix(IList<TaskStats> taskStats)
+ {
+ string longestOp = LongestOp(taskStats);
+ bool first = true;
+ StringBuilder sb = new StringBuilder();
+ sb.Append(TableTitle(longestOp));
+ sb.Append(newline);
+ int reported = 0;
+ foreach (TaskStats stat in taskStats)
+ {
+ if (stat.Elapsed >= 0 && stat.Task.GetName().StartsWith(m_prefix, StringComparison.Ordinal))
+ { // only ended tasks with proper name
+ reported++;
+ if (!first)
+ {
+ sb.Append(newline);
+ }
+ first = false;
+ string line = TaskReportLine(longestOp, stat);
+ if (taskStats.Count > 2 && reported % 2 == 0)
+ {
+ line = line.Replace(" ", " - ");
+ }
+ sb.Append(line);
+ }
+ }
+ string reptxt = (reported == 0 ? "No Matching Entries Were Found!" : sb.ToString());
+ return new Report(reptxt, reported, reported, taskStats.Count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameRoundTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameRoundTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameRoundTask.cs
new file mode 100644
index 0000000..bce47ca
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameRoundTask.cs
@@ -0,0 +1,83 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report all statistics grouped/aggregated by name and round.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class RepSumByNameRoundTask : ReportTask
+ {
+ public RepSumByNameRoundTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ Report rp = ReportSumByNameRound(RunData.Points.TaskStats);
+
+ SystemConsole.WriteLine();
+ SystemConsole.WriteLine("------------> Report Sum By (any) Name and Round (" +
+ rp.Count + " about " + rp.Reported + " out of " + rp.OutOf + ")");
+ SystemConsole.WriteLine(rp.Text);
+ SystemConsole.WriteLine();
+
+ return 0;
+ }
+
+ /// <summary>
+ /// Report statistics as a string, aggregate for tasks named the same, and from the same round.
+ /// </summary>
+ /// <param name="taskStats"></param>
+ /// <returns>The report.</returns>
+ protected virtual Report ReportSumByNameRound(IList<TaskStats> taskStats)
+ {
+ // aggregate by task name and round
+ LinkedHashMap<string, TaskStats> p2 = new LinkedHashMap<string, TaskStats>();
+ int reported = 0;
+ foreach (TaskStats stat1 in taskStats)
+ {
+ if (stat1.Elapsed >= 0)
+ { // consider only tasks that ended
+ reported++;
+ string name = stat1.Task.GetName();
+ string rname = stat1.Round + "." + name; // group by round
+ TaskStats stat2;
+ if (!p2.TryGetValue(rname, out stat2) || stat2 == null)
+ {
+ stat2 = (TaskStats)stat1.Clone();
+
+ p2[rname] = stat2;
+ }
+ else
+ {
+ stat2.Add(stat1);
+ }
+ }
+ }
+ // now generate report from secondary list p2
+ return GenPartialReport(reported, p2, taskStats.Count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameTask.cs
new file mode 100644
index 0000000..30d9878
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByNameTask.cs
@@ -0,0 +1,81 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report all statistics aggregated by name.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class RepSumByNameTask : ReportTask
+ {
+ public RepSumByNameTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ Report rp = ReportSumByName(RunData.Points.TaskStats);
+
+ SystemConsole.WriteLine();
+ SystemConsole.WriteLine("------------> Report Sum By (any) Name (" +
+ rp.Count + " about " + rp.Reported + " out of " + rp.OutOf + ")");
+ SystemConsole.WriteLine(rp.Text);
+ SystemConsole.WriteLine();
+
+ return 0;
+ }
+
+ /// <summary>
+ /// Report statistics as a string, aggregate for tasks named the same.
+ /// </summary>
+ /// <param name="taskStats"></param>
+ /// <returns>The report.</returns>
+ protected virtual Report ReportSumByName(IList<TaskStats> taskStats)
+ {
+ // aggregate by task name
+ int reported = 0;
+ LinkedHashMap<string, TaskStats> p2 = new LinkedHashMap<string, TaskStats>();
+ foreach (TaskStats stat1 in taskStats)
+ {
+ if (stat1.Elapsed >= 0)
+ { // consider only tasks that ended
+ reported++;
+ string name = stat1.Task.GetName();
+ TaskStats stat2;
+ if (!p2.TryGetValue(name, out stat2) || stat2 == null)
+ {
+ stat2 = (TaskStats)stat1.Clone();
+ p2[name] = stat2;
+ }
+ else
+ {
+ stat2.Add(stat1);
+ }
+ }
+ }
+ // now generate report from secondary list p2
+ return GenPartialReport(reported, p2, taskStats.Count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefRoundTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefRoundTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefRoundTask.cs
new file mode 100644
index 0000000..ce977a0
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefRoundTask.cs
@@ -0,0 +1,79 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report all prefix matching statistics grouped/aggregated by name and round.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class RepSumByPrefRoundTask : RepSumByPrefTask
+ {
+ public RepSumByPrefRoundTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ Report rp = ReportSumByPrefixRound(RunData.Points.TaskStats);
+
+ Console.Out.WriteLine();
+ Console.Out.WriteLine("------------> Report sum by Prefix (" + m_prefix + ") and Round (" +
+ rp.Count + " about " + rp.Reported + " out of " + rp.OutOf + ")");
+ Console.Out.WriteLine(rp.Text);
+ Console.Out.WriteLine();
+
+ return 0;
+ }
+
+ protected virtual Report ReportSumByPrefixRound(IList<TaskStats> taskStats)
+ {
+ // aggregate by task name and by round
+ int reported = 0;
+ LinkedHashMap<string, TaskStats> p2 = new LinkedHashMap<string, TaskStats>();
+ foreach (TaskStats stat1 in taskStats)
+ {
+ if (stat1.Elapsed >= 0 && stat1.Task.GetName().StartsWith(m_prefix, StringComparison.Ordinal))
+ { // only ended tasks with proper name
+ reported++;
+ string name = stat1.Task.GetName();
+ string rname = stat1.Round + "." + name; // group by round
+ TaskStats stat2;
+ if (!p2.TryGetValue(rname, out stat2) || stat2 == null)
+ {
+ stat2 = (TaskStats)stat1.Clone();
+
+ p2[rname] = stat2;
+ }
+ else
+ {
+ stat2.Add(stat1);
+ }
+ }
+ }
+ // now generate report from secondary list p2
+ return GenPartialReport(reported, p2, taskStats.Count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefTask.cs
new file mode 100644
index 0000000..732e902
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RepSumByPrefTask.cs
@@ -0,0 +1,91 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report by-name-prefix statistics aggregated by name.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class RepSumByPrefTask : ReportTask
+ {
+ public RepSumByPrefTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ protected string m_prefix;
+
+ public override int DoLogic()
+ {
+ Report rp = ReportSumByPrefix(RunData.Points.TaskStats);
+
+ SystemConsole.WriteLine();
+ SystemConsole.WriteLine("------------> Report Sum By Prefix (" + m_prefix + ") (" +
+ rp.Count + " about " + rp.Reported + " out of " + rp.OutOf + ")");
+ SystemConsole.WriteLine(rp.Text);
+ SystemConsole.WriteLine();
+
+ return 0;
+ }
+
+ protected virtual Report ReportSumByPrefix(IList<TaskStats> taskStats)
+ {
+ // aggregate by task name
+ int reported = 0;
+ LinkedHashMap<string, TaskStats> p2 = new LinkedHashMap<string, TaskStats>();
+ foreach (TaskStats stat1 in taskStats)
+ {
+ if (stat1.Elapsed >= 0 && stat1.Task.GetName().StartsWith(m_prefix, StringComparison.Ordinal))
+ { // only ended tasks with proper name
+ reported++;
+ string name = stat1.Task.GetName();
+ TaskStats stat2;
+ if (!p2.TryGetValue(name, out stat2) || stat2 == null)
+ {
+ stat2 = (TaskStats)stat1.Clone();
+ p2[name] = stat2;
+ }
+ else
+ {
+ stat2.Add(stat1);
+ }
+ }
+ }
+ // now generate report from secondary list p2
+ return GenPartialReport(reported, p2, taskStats.Count);
+ }
+
+
+ public virtual void SetPrefix(string prefix)
+ {
+ this.m_prefix = prefix;
+ }
+
+ /// <seealso cref="PerfTask.ToString()"/>
+ public override string ToString()
+ {
+ return base.ToString() + " " + m_prefix;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ReportTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ReportTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReportTask.cs
new file mode 100644
index 0000000..0e83471
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReportTask.cs
@@ -0,0 +1,189 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Report (abstract) task - all report tasks extend this task.
+ /// </summary>
+ public abstract class ReportTask : PerfTask
+ {
+ public ReportTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ /// <seealso cref="PerfTask.ShouldNeverLogAtStart"/>
+ protected override bool ShouldNeverLogAtStart
+ {
+ get { return true; }
+ }
+
+ /// <seealso cref="PerfTask.ShouldNotRecordStats"/>
+ protected override bool ShouldNotRecordStats
+ {
+ get { return true; }
+ }
+
+ /// <summary>
+ /// From here start the code used to generate the reports.
+ /// Subclasses would use this part to generate reports.
+ /// </summary>
+ protected static readonly string newline = Environment.NewLine;
+
+ /// <summary>
+ /// Get a textual summary of the benchmark results, average from all test runs.
+ /// </summary>
+ protected static readonly string OP = "Operation ";
+ protected static readonly string ROUND = " round";
+ protected static readonly string RUNCNT = " runCnt";
+ protected static readonly string RECCNT = " recsPerRun";
+ protected static readonly string RECSEC = " rec/s";
+ protected static readonly string ELAPSED = " elapsedSec";
+ protected static readonly string USEDMEM = " avgUsedMem";
+ protected static readonly string TOTMEM = " avgTotalMem";
+ protected static readonly string[] COLS = {
+ RUNCNT,
+ RECCNT,
+ RECSEC,
+ ELAPSED,
+ USEDMEM,
+ TOTMEM
+ };
+
+ /// <summary>
+ /// Compute a title line for a report table.
+ /// </summary>
+ /// <param name="longestOp">Size of longest op name in the table.</param>
+ /// <returns>The table title line.</returns>
+ protected virtual string TableTitle(string longestOp)
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.Append(Formatter.Format(OP, longestOp));
+ sb.Append(ROUND);
+ sb.Append(RunData.Config.GetColsNamesForValsByRound());
+ for (int i = 0; i < COLS.Length; i++)
+ {
+ sb.Append(COLS[i]);
+ }
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Find the longest op name out of completed tasks.
+ /// </summary>
+ /// <param name="taskStats">Completed tasks to be considered.</param>
+ /// <returns>The longest op name out of completed tasks.</returns>
+ protected virtual string LongestOp(IEnumerable<TaskStats> taskStats)
+ {
+ string longest = OP;
+ foreach (TaskStats stat in taskStats)
+ {
+ if (stat.Elapsed >= 0)
+ { // consider only tasks that ended
+ string name = stat.Task.GetName();
+ if (name.Length > longest.Length)
+ {
+ longest = name;
+ }
+ }
+ }
+ return longest;
+ }
+
+ /// <summary>
+ /// Compute a report line for the given task stat.
+ /// </summary>
+ /// <param name="longestOp">Size of longest op name in the table.</param>
+ /// <param name="stat">Task stat to be printed.</param>
+ /// <returns>The report line.</returns>
+ protected virtual string TaskReportLine(string longestOp, TaskStats stat)
+ {
+ PerfTask task = stat.Task;
+ StringBuilder sb = new StringBuilder();
+ sb.Append(Formatter.Format(task.GetName(), longestOp));
+ string round = (stat.Round >= 0 ? "" + stat.Round : "-");
+ sb.Append(Formatter.FormatPaddLeft(round, ROUND));
+ sb.Append(RunData.Config.GetColsValuesForValsByRound(stat.Round));
+ sb.Append(Formatter.Format(stat.NumRuns, RUNCNT));
+ sb.Append(Formatter.Format(stat.Count / stat.NumRuns, RECCNT));
+ long elapsed = (stat.Elapsed > 0 ? stat.Elapsed : 1); // assume at least 1ms
+ sb.Append(Formatter.Format(2, (float)(stat.Count * 1000.0 / elapsed), RECSEC));
+ sb.Append(Formatter.Format(2, (float)stat.Elapsed / 1000, ELAPSED));
+ sb.Append(Formatter.Format(0, (float)stat.MaxUsedMem / stat.NumRuns, USEDMEM));
+ sb.Append(Formatter.Format(0, (float)stat.MaxTotMem / stat.NumRuns, TOTMEM));
+ return sb.ToString();
+ }
+
+ protected virtual Report GenPartialReport(int reported, LinkedHashMap<string, TaskStats> partOfTasks, int totalSize)
+ {
+ string longetOp = LongestOp(partOfTasks.Values);
+ bool first = true;
+ StringBuilder sb = new StringBuilder();
+ sb.Append(TableTitle(longetOp));
+ sb.Append(newline);
+ int lineNum = 0;
+ foreach (TaskStats stat in partOfTasks.Values)
+ {
+ if (!first)
+ {
+ sb.Append(newline);
+ }
+ first = false;
+ string line = TaskReportLine(longetOp, stat);
+ lineNum++;
+ if (partOfTasks.Count > 2 && lineNum % 2 == 0)
+ {
+ line = line.Replace(" ", " - ");
+ }
+ sb.Append(line);
+ int[] byTime = stat.GetCountsByTime();
+ if (byTime != null)
+ {
+ sb.Append(newline);
+ int end = -1;
+ for (int i = byTime.Length - 1; i >= 0; i--)
+ {
+ if (byTime[i] != 0)
+ {
+ end = i;
+ break;
+ }
+ }
+ if (end != -1)
+ {
+ sb.Append(" by time:");
+ for (int i = 0; i < end; i++)
+ {
+ sb.Append(' ').Append(byTime[i]);
+ }
+ }
+ }
+ }
+
+ string reptxt = (reported == 0 ? "No Matching Entries Were Found!" : sb.ToString());
+ return new Report(reptxt, partOfTasks.Count, reported, totalSize);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetInputsTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetInputsTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetInputsTask.cs
new file mode 100644
index 0000000..ed6261f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetInputsTask.cs
@@ -0,0 +1,43 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Reset inputs so that the test run would behave, input wise,
+ /// as if it just started. This affects e.g. the generation of docs and queries.
+ /// </summary>
+ public class ResetInputsTask : PerfTask
+ {
+ public ResetInputsTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.ResetInputs();
+ return 0;
+ }
+
+ /// <seealso cref="PerfTask.ShouldNotRecordStats"/>
+ protected override bool ShouldNotRecordStats
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemEraseTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemEraseTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemEraseTask.cs
new file mode 100644
index 0000000..b8e854a
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemEraseTask.cs
@@ -0,0 +1,42 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Reset all index and input data and call gc, erase index and dir, does NOT clear statistics.
+ /// <para/>
+ /// This contains ResetInputs.
+ /// <para/>
+ /// Other side effects: writers/readers nullified, deleted, closed.
+ /// Index is erased.
+ /// Directory is erased.
+ /// </summary>
+ public class ResetSystemEraseTask : ResetSystemSoftTask
+ {
+ public ResetSystemEraseTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.Reinit(true);
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemSoftTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemSoftTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemSoftTask.cs
new file mode 100644
index 0000000..5d2d4f5
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ResetSystemSoftTask.cs
@@ -0,0 +1,41 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Reset all index and input data and call gc, does NOT erase index/dir, does NOT clear statistics.
+ /// This contains ResetInputs.
+ /// <para/>
+ /// Other side effects: writers/readers nullified, closed.
+ /// Index is NOT erased.
+ /// Directory is NOT erased.
+ /// </summary>
+ public class ResetSystemSoftTask : ResetInputsTask
+ {
+ public ResetSystemSoftTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.Reinit(false);
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/RollbackIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/RollbackIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/RollbackIndexTask.cs
new file mode 100644
index 0000000..c189b93
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/RollbackIndexTask.cs
@@ -0,0 +1,52 @@
+using Lucene.Net.Index;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Rollback the index writer.
+ /// </summary>
+ public class RollbackIndexTask : PerfTask
+ {
+ public RollbackIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ //bool doWait = true; // LUCENENET: Not used
+
+ public override int DoLogic()
+ {
+ IndexWriter iw = RunData.IndexWriter;
+ if (iw != null)
+ {
+ // If infoStream was set to output to a file, close it.
+ InfoStream infoStream = iw.Config.InfoStream;
+ if (infoStream != null)
+ {
+ infoStream.Dispose();
+ }
+ iw.Rollback();
+ RunData.IndexWriter = null;
+ }
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTask.cs
new file mode 100644
index 0000000..f7dd4b7
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTask.cs
@@ -0,0 +1,60 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Search task.
+ /// <para/>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// </summary>
+ public class SearchTask : ReadTask
+ {
+ public SearchTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return false; }
+ }
+
+ public override bool WithSearch
+ {
+ get { return true; }
+ }
+
+ public override bool WithTraverse
+ {
+ get { return false; }
+ }
+
+ public override bool WithWarm
+ {
+ get { return false; }
+ }
+
+ public override IQueryMaker GetQueryMaker()
+ {
+ return RunData.GetQueryMaker(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetHighlightTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetHighlightTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetHighlightTask.cs
new file mode 100644
index 0000000..2d623b1
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetHighlightTask.cs
@@ -0,0 +1,188 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Highlight;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Search and Traverse and Retrieve docs task. Highlight the fields in the retrieved documents.
+ /// </summary>
+ /// <remarks>
+ /// Uses the <see cref="SimpleHTMLFormatter"/> for formatting.
+ /// <para/>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Takes optional multivalued, comma separated param string as:
+ /// <code>
+ /// size[<traversal size>],highlight[<int>],maxFrags[<int>],mergeContiguous[<boolean>],fields[name1;name2;...]
+ /// </code>
+ /// <list type="table">
+ /// <item><term>traversal size</term><description>The number of hits to traverse, otherwise all will be traversed.</description></item>
+ /// <item><term>highlight</term><description>The number of the hits to highlight. Will always be less than or equal to traversal size. Default is <see cref="int.MaxValue"/> (i.e. hits.Length).</description></item>
+ /// <item><term>maxFrags</term><description>The maximum number of fragments to score by the highlighter.</description></item>
+ /// <item><term>mergeContiguous</term><description><c>true</c> if contiguous fragments should be merged.</description></item>
+ /// <item><term>fields</term><description>The fields to highlight. If not specified all fields will be highlighted (or at least attempted).</description></item>
+ /// </list>
+ /// <para/>
+ /// Example:
+ /// <code>
+ /// "SearchHlgtSameRdr" SearchTravRetHighlight(size[10],highlight[10],mergeContiguous[true],maxFrags[3],fields[body]) > : 1000
+ /// </code>
+ /// <para/>
+ /// Documents must be stored in order for this task to work. Additionally, term vector positions can be used as well.
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each traversed hit,
+ /// and 1 more for each retrieved (non null) document and 1 for each fragment returned.
+ /// </remarks>
+ public class SearchTravRetHighlightTask : SearchTravTask
+ {
+ protected int m_numToHighlight = int.MaxValue;
+ protected bool m_mergeContiguous;
+ protected int m_maxFrags = 2;
+ protected ISet<string> m_paramFields = new HashSet<string>();
+ protected Highlighter m_highlighter;
+ protected int m_maxDocCharsToAnalyze;
+
+ public SearchTravRetHighlightTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void Setup()
+ {
+ base.Setup();
+ //check to make sure either the doc is being stored
+ PerfRunData data = RunData;
+ if (data.Config.Get("doc.stored", false) == false)
+ {
+ throw new Exception("doc.stored must be set to true");
+ }
+ m_maxDocCharsToAnalyze = data.Config.Get("highlighter.maxDocCharsToAnalyze", Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE);
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return true; }
+ }
+
+ public override int NumToHighlight
+ {
+ get { return m_numToHighlight; }
+ }
+
+ protected override BenchmarkHighlighter GetBenchmarkHighlighter(Query q)
+ {
+ m_highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
+ m_highlighter.MaxDocCharsToAnalyze = m_maxDocCharsToAnalyze;
+ return new BenchmarkHighlighterAnonymousHelper(this, m_highlighter);
+ }
+
+ private class BenchmarkHighlighterAnonymousHelper : BenchmarkHighlighter
+ {
+ private readonly SearchTravRetHighlightTask outerInstance;
+ private readonly Highlighter highlighter;
+
+ public BenchmarkHighlighterAnonymousHelper(SearchTravRetHighlightTask outerInstance, Highlighter highlighter)
+ {
+ this.outerInstance = outerInstance;
+ this.highlighter = highlighter;
+ }
+ public override int DoHighlight(IndexReader reader, int doc, string field, Document document, Analyzer analyzer, string text)
+ {
+ TokenStream ts = TokenSources.GetAnyTokenStream(reader, doc, field, document, analyzer);
+ TextFragment[]
+ frag = highlighter.GetBestTextFragments(ts, text, outerInstance.m_mergeContiguous, outerInstance.m_maxFrags);
+ return frag != null ? frag.Length : 0;
+ }
+ }
+
+ protected override ICollection<string> GetFieldsToHighlight(Document document)
+ {
+ ICollection<string> result = base.GetFieldsToHighlight(document);
+ //if stored is false, then result will be empty, in which case just get all the param fields
+ if (m_paramFields.Count > 0 && result.Count > 0)
+ {
+ //result.RetainAll(paramFields);
+ var toRemove = new List<string>();
+ foreach (var e in result)
+ {
+ if (!m_paramFields.Contains(e))
+ toRemove.Add(e);
+ }
+ result.RemoveAll(toRemove);
+ }
+ else
+ {
+ result = m_paramFields;
+ }
+ return result;
+ }
+
+ public override void SetParams(string @params)
+ {
+ // can't call super because super doesn't understand our
+ // params syntax
+ this.m_params = @params;
+ string[] splits = @params.Split(',').TrimEnd();
+ for (int i = 0; i < splits.Length; i++)
+ {
+ if (splits[i].StartsWith("size[", StringComparison.Ordinal) == true)
+ {
+ int len = "size[".Length;
+ m_traversalSize = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("highlight[", StringComparison.Ordinal) == true)
+ {
+ int len = "highlight[".Length;
+ m_numToHighlight = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("maxFrags[", StringComparison.Ordinal) == true)
+ {
+ int len = "maxFrags[".Length;
+ m_maxFrags = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("mergeContiguous[", StringComparison.Ordinal) == true)
+ {
+ int len = "mergeContiguous[".Length;
+ m_mergeContiguous = bool.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len));
+ }
+ else if (splits[i].StartsWith("fields[", StringComparison.Ordinal) == true)
+ {
+ m_paramFields = new HashSet<string>();
+ int len = "fields[".Length;
+ string fieldNames = splits[i].Substring(len, (splits[i].Length - 1) - len);
+ string[] fieldSplits = fieldNames.Split(';').TrimEnd();
+ for (int j = 0; j < fieldSplits.Length; j++)
+ {
+ m_paramFields.Add(fieldSplits[j]);
+ }
+
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetLoadFieldSelectorTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetLoadFieldSelectorTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetLoadFieldSelectorTask.cs
new file mode 100644
index 0000000..169a330
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetLoadFieldSelectorTask.cs
@@ -0,0 +1,85 @@
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Search and Traverse and Retrieve docs task using a
+ /// FieldVisitor loading only the requested fields.
+ /// </summary>
+ /// <remarks>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Takes optional param: comma separated list of Fields to load.
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each traversed hit,
+ /// and 1 more for each retrieved (non null) document.
+ /// </remarks>
+ public class SearchTravRetLoadFieldSelectorTask : SearchTravTask
+ {
+ protected ISet<string> m_fieldsToLoad;
+
+ public SearchTravRetLoadFieldSelectorTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return true; }
+ }
+
+
+ protected override Document RetrieveDoc(IndexReader ir, int id)
+ {
+ if (m_fieldsToLoad == null)
+ {
+ return ir.Document(id);
+ }
+ else
+ {
+ DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(m_fieldsToLoad);
+ ir.Document(id, visitor);
+ return visitor.Document;
+ }
+ }
+
+ public override void SetParams(string @params)
+ {
+ this.m_params = @params; // cannot just call super.setParams(), b/c it's params differ.
+ m_fieldsToLoad = new HashSet<string>();
+ for (StringTokenizer tokenizer = new StringTokenizer(@params, ","); tokenizer.HasMoreTokens();)
+ {
+ string s = tokenizer.NextToken();
+ m_fieldsToLoad.Add(s);
+ }
+ }
+
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetTask.cs
new file mode 100644
index 0000000..61d1436
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetTask.cs
@@ -0,0 +1,44 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Search and Traverse and Retrieve docs task.
+ /// </summary>
+ /// <remarks>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Takes optional param: traversal size (otherwise all results are traversed).
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each traversed hit,
+ /// and 1 more for each retrieved (non null) document.
+ /// </remarks>
+ public class SearchTravRetTask : SearchTravTask
+ {
+ public SearchTravRetTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetVectorHighlightTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetVectorHighlightTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetVectorHighlightTask.cs
new file mode 100644
index 0000000..0725c65
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravRetVectorHighlightTask.cs
@@ -0,0 +1,191 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Search.VectorHighlight;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Search and Traverse and Retrieve docs task. Highlight the fields in the retrieved documents by using <see cref="FastVectorHighlighter"/>.
+ /// </summary>
+ /// <remarks>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Takes optional multivalued, comma separated param string as:
+ /// <code>
+ /// size[<traversal size>],highlight[<int>],maxFrags[<int>],mergeContiguous[<boolean>],fields[name1;name2;...]
+ /// </code>
+ /// <list type="bullet">
+ /// <item><term>traversal size</term><description>The number of hits to traverse, otherwise all will be traversed.</description></item>
+ /// <item><term>highlight</term><description>The number of the hits to highlight. Will always be less than or equal to traversal size. Default is <see cref="int.MaxValue"/> (i.e. hits.Length).</description></item>
+ /// <item><term>maxFrags</term><description>The maximum number of fragments to score by the highlighter.</description></item>
+ /// <item><term>fragSize</term><description>The length of fragments.</description></item>
+ /// <item><term>fields</term><description>The fields to highlight. If not specified all fields will be highlighted (or at least attempted).</description></item>
+ /// </list>
+ /// <para/>
+ /// Example:
+ /// <code>
+ /// "SearchVecHlgtSameRdr" SearchTravRetVectorHighlight(size[10],highlight[10],maxFrags[3],fields[body]) > : 1000
+ /// </code>
+ /// <para/>
+ /// Fields must be stored and term vector offsets and positions in order must be true for this task to work.
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each traversed hit,
+ /// and 1 more for each retrieved (non null) document and 1 for each fragment returned.
+ /// </remarks>
+ public class SearchTravRetVectorHighlightTask : SearchTravTask
+ {
+ protected int m_numToHighlight = int.MaxValue;
+ protected int m_maxFrags = 2;
+ protected int m_fragSize = 100;
+ protected ISet<string> m_paramFields = new HashSet<string>();
+ protected FastVectorHighlighter m_highlighter;
+
+ public SearchTravRetVectorHighlightTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void Setup()
+ {
+ base.Setup();
+ //check to make sure either the doc is being stored
+ PerfRunData data = RunData;
+ if (data.Config.Get("doc.stored", false) == false)
+ {
+ throw new Exception("doc.stored must be set to true");
+ }
+ if (data.Config.Get("doc.term.vector.offsets", false) == false)
+ {
+ throw new Exception("doc.term.vector.offsets must be set to true");
+ }
+ if (data.Config.Get("doc.term.vector.positions", false) == false)
+ {
+ throw new Exception("doc.term.vector.positions must be set to true");
+ }
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return true; }
+ }
+
+ public override int NumToHighlight
+ {
+ get { return m_numToHighlight; }
+ }
+
+ protected override BenchmarkHighlighter GetBenchmarkHighlighter(Query q)
+ {
+ m_highlighter = new FastVectorHighlighter(false, false);
+ Query myq = q;
+ return new BenchmarkHighlighterAnonymousHelper(this, m_highlighter, myq);
+ }
+
+ private class BenchmarkHighlighterAnonymousHelper : BenchmarkHighlighter
+ {
+ private readonly SearchTravRetVectorHighlightTask outerInstance;
+ private readonly FastVectorHighlighter highlighter;
+ private readonly Query myq;
+ public BenchmarkHighlighterAnonymousHelper(SearchTravRetVectorHighlightTask outerInstance, FastVectorHighlighter highlighter, Query myq)
+ {
+ this.outerInstance = outerInstance;
+ this.highlighter = highlighter;
+ this.myq = myq;
+ }
+ public override int DoHighlight(IndexReader reader, int doc, string field, Document document, Analyzer analyzer, string text)
+ {
+ FieldQuery fq = highlighter.GetFieldQuery(myq, reader);
+ string[] fragments = highlighter.GetBestFragments(fq, reader, doc, field, outerInstance.m_fragSize, outerInstance.m_maxFrags);
+ return fragments != null ? fragments.Length : 0;
+ }
+ }
+
+ protected override ICollection<string> GetFieldsToHighlight(Document document)
+ {
+ ICollection<string> result = base.GetFieldsToHighlight(document);
+ //if stored is false, then result will be empty, in which case just get all the param fields
+ if (m_paramFields.Count > 0 && result.Count > 0)
+ {
+ //result.RetainAll(paramFields);
+ var toRemove = new List<string>();
+ foreach (var e in result)
+ {
+ if (!m_paramFields.Contains(e))
+ toRemove.Add(e);
+ }
+ result.RemoveAll(toRemove);
+ }
+ else
+ {
+ result = m_paramFields;
+ }
+ return result;
+ }
+
+ public override void SetParams(string @params)
+ {
+ // can't call super because super doesn't understand our
+ // params syntax
+ string[] splits = @params.Split(',').TrimEnd();
+ for (int i = 0; i < splits.Length; i++)
+ {
+ if (splits[i].StartsWith("size[", StringComparison.Ordinal) == true)
+ {
+ int len = "size[".Length;
+ m_traversalSize = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("highlight[", StringComparison.Ordinal) == true)
+ {
+ int len = "highlight[".Length;
+ m_numToHighlight = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("maxFrags[", StringComparison.Ordinal) == true)
+ {
+ int len = "maxFrags[".Length;
+ m_maxFrags = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("fragSize[", StringComparison.Ordinal) == true)
+ {
+ int len = "fragSize[".Length;
+ m_fragSize = (int)float.Parse(splits[i].Substring(len, (splits[i].Length - 1) - len), CultureInfo.InvariantCulture);
+ }
+ else if (splits[i].StartsWith("fields[", StringComparison.Ordinal) == true)
+ {
+ m_paramFields = new HashSet<string>();
+ int len = "fields[".Length;
+ string fieldNames = splits[i].Substring(len, (splits[i].Length - 1) - len);
+ string[] fieldSplits = fieldNames.Split(';').TrimEnd();
+ for (int j = 0; j < fieldSplits.Length; j++)
+ {
+ m_paramFields.Add(fieldSplits[j]);
+ }
+
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravTask.cs
new file mode 100644
index 0000000..a514411
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchTravTask.cs
@@ -0,0 +1,87 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Search and Traverse task.
+ /// </summary>
+ /// <remarks>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Takes optional param: traversal size (otherwise all results are traversed).
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each traversed hit.
+ /// </remarks>
+ public class SearchTravTask : ReadTask
+ {
+ protected int m_traversalSize = int.MaxValue;
+
+ public SearchTravTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return false; }
+ }
+
+
+ public override bool WithSearch
+ {
+ get { return true; }
+ }
+
+ public override bool WithTraverse
+ {
+ get { return true; }
+ }
+
+ public override bool WithWarm
+ {
+ get { return false; }
+ }
+
+
+ public override IQueryMaker GetQueryMaker()
+ {
+ return RunData.GetQueryMaker(this);
+ }
+
+ public override int TraversalSize
+ {
+ get { return m_traversalSize; }
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ m_traversalSize = (int)float.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithCollectorTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithCollectorTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithCollectorTask.cs
new file mode 100644
index 0000000..b767b44
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithCollectorTask.cs
@@ -0,0 +1,99 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Search;
+using System;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Does search w/ a custom collector
+ /// </summary>
+ public class SearchWithCollectorTask : SearchTask
+ {
+ protected string m_clnName;
+
+ public SearchWithCollectorTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void Setup()
+ {
+ base.Setup();
+ //check to make sure either the doc is being stored
+ PerfRunData runData = RunData;
+ Config config = runData.Config;
+ m_clnName = config.Get("collector.class", "");
+ }
+
+ public override bool WithCollector
+ {
+ get { return true; }
+ }
+
+ protected override ICollector CreateCollector()
+ {
+ ICollector collector = null;
+ if (m_clnName.Equals("topScoreDocOrdered", StringComparison.OrdinalIgnoreCase) == true)
+ {
+ collector = TopScoreDocCollector.Create(NumHits, true);
+ }
+ else if (m_clnName.Equals("topScoreDocUnOrdered", StringComparison.OrdinalIgnoreCase) == true)
+ {
+ collector = TopScoreDocCollector.Create(NumHits, false);
+ }
+ else if (m_clnName.Length > 0)
+ {
+ collector = (ICollector)Activator.CreateInstance(Type.GetType(m_clnName));
+
+ }
+ else
+ {
+ collector = base.CreateCollector();
+ }
+ return collector;
+ }
+
+ public override IQueryMaker GetQueryMaker()
+ {
+ return RunData.GetQueryMaker(this);
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return false; }
+ }
+
+ public override bool WithSearch
+ {
+ get { return true; }
+ }
+
+ public override bool WithTraverse
+ {
+ get { return false; }
+ }
+
+ public override bool WithWarm
+ {
+ get { return false; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithSortTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithSortTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithSortTask.cs
new file mode 100644
index 0000000..634f048
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SearchWithSortTask.cs
@@ -0,0 +1,157 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using System;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Does sort search on specified field.
+ /// </summary>
+ public class SearchWithSortTask : ReadTask
+ {
+ private bool doScore = true;
+ private bool doMaxScore = true;
+ private Sort sort;
+
+ public SearchWithSortTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ /// <summary>
+ /// SortFields: field:type,field:type[,noscore][,nomaxscore]
+ /// <para/>
+ /// If noscore is present, then we turn off score tracking
+ /// in <see cref="TopFieldCollector"/>.
+ /// If nomaxscore is present, then we turn off maxScore tracking
+ /// in <see cref="TopFieldCollector"/>.
+ /// <para/>
+ /// name:string,page:int,subject:string
+ /// </summary>
+ public override void SetParams(string sortField)
+ {
+ base.SetParams(sortField);
+ string[] fields = sortField.Split(',').TrimEnd();
+ SortField[] sortFields = new SortField[fields.Length];
+ int upto = 0;
+ for (int i = 0; i < fields.Length; i++)
+ {
+ string field = fields[i];
+ SortField sortField0;
+ if (field.Equals("doc", StringComparison.Ordinal))
+ {
+ sortField0 = SortField.FIELD_DOC;
+ }
+ else if (field.Equals("score", StringComparison.Ordinal))
+ {
+ sortField0 = SortField.FIELD_SCORE;
+ }
+ else if (field.Equals("noscore", StringComparison.Ordinal))
+ {
+ doScore = false;
+ continue;
+ }
+ else if (field.Equals("nomaxscore", StringComparison.Ordinal))
+ {
+ doMaxScore = false;
+ continue;
+ }
+ else
+ {
+ int index = field.LastIndexOf(':');
+ string fieldName;
+ string typeString;
+ if (index != -1)
+ {
+ fieldName = field.Substring(0, index - 0);
+ typeString = field.Substring(1 + index, field.Length - (1 + index));
+ }
+ else
+ {
+ throw new Exception("You must specify the sort type ie page:int,subject:string");
+ }
+ sortField0 = new SortField(fieldName, (SortFieldType)Enum.Parse(typeof(SortFieldType), typeString, true));
+ }
+ sortFields[upto++] = sortField0;
+ }
+
+ if (upto < sortFields.Length)
+ {
+ SortField[] newSortFields = new SortField[upto];
+ System.Array.Copy(sortFields, 0, newSortFields, 0, upto);
+ sortFields = newSortFields;
+ }
+ this.sort = new Sort(sortFields);
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+
+ public override IQueryMaker GetQueryMaker()
+ {
+ return RunData.GetQueryMaker(this);
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return false; }
+ }
+
+ public override bool WithSearch
+ {
+ get { return true; }
+ }
+
+ public override bool WithTraverse
+ {
+ get { return false; }
+ }
+
+ public override bool WithWarm
+ {
+ get { return false; }
+ }
+
+ public override bool WithScore
+ {
+ get { return doScore; }
+ }
+
+ public override bool WithMaxScore
+ {
+ get { return doMaxScore; }
+ }
+
+ public override Sort Sort
+ {
+ get
+ {
+ if (sort == null)
+ {
+ throw new InvalidOperationException("No sort field was set");
+ }
+ return sort;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/SetPropTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/SetPropTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/SetPropTask.cs
new file mode 100644
index 0000000..abb66d2
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/SetPropTask.cs
@@ -0,0 +1,71 @@
+using System;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Set a performance test configuration property.
+ /// A property may have a single value, or a sequence of values, separated by ":".
+ /// If a sequence of values is specified, each time a new round starts,
+ /// the next (cyclic) value is taken.
+ /// <para/>
+ /// Other side effects: none.
+ /// <para/>
+ /// Takes mandatory param: "name,value" pair.
+ /// </summary>
+ /// <seealso cref="NewRoundTask"/>
+ public class SetPropTask : PerfTask
+ {
+ public SetPropTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ private string name;
+ private string value;
+
+ public override int DoLogic()
+ {
+ if (name == null || value == null)
+ {
+ throw new Exception(GetName() + " - undefined name or value: name=" + name + " value=" + value);
+ }
+ RunData.Config.Set(name, value);
+ return 0;
+ }
+
+ /// <summary>
+ /// Set the params (property name and value).
+ /// </summary>
+ /// <param name="params">Property name and value separated by ','.</param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ int k = @params.IndexOf(",");
+ name = @params.Substring(0, k - 0).Trim();
+ value = @params.Substring(k + 1).Trim();
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
[12/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
Ported Lucene.Net.Benchmark + tests
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/b515271d
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/b515271d
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/b515271d
Branch: refs/heads/master
Commit: b515271d8821dde3cd980beae780d204fd6b0e5c
Parents: 1e52293
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Mon Jul 31 14:26:48 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Wed Aug 2 09:54:52 2017 +0700
----------------------------------------------------------------------
Lucene.Net.sln | 52 +
src/Lucene.Net.Benchmark/ByTask/Benchmark.cs | 170 +++
.../ByTask/Feeds/AbstractQueryMaker.cs | 85 ++
.../ByTask/Feeds/ContentItemsSource.cs | 227 ++++
.../ByTask/Feeds/ContentSource.cs | 38 +
.../ByTask/Feeds/DemoHTMLParser.cs | 259 ++++
.../ByTask/Feeds/DirContentSource.cs | 259 ++++
.../ByTask/Feeds/DocData.cs | 73 ++
.../ByTask/Feeds/DocMaker.cs | 511 ++++++++
.../ByTask/Feeds/EnwikiContentSource.cs | 394 ++++++
.../ByTask/Feeds/EnwikiQueryMaker.cs | 146 +++
.../ByTask/Feeds/FacetSource.cs | 47 +
.../ByTask/Feeds/FileBasedQueryMaker.cs | 121 ++
.../ByTask/Feeds/GeonamesLineParser.cs | 53 +
.../ByTask/Feeds/HTMLParser.cs | 42 +
.../ByTask/Feeds/LineDocSource.cs | 328 +++++
.../ByTask/Feeds/LongToEnglishContentSource.cs | 72 ++
.../ByTask/Feeds/LongToEnglishQueryMaker.cs | 89 ++
.../ByTask/Feeds/NoMoreDataException.cs | 50 +
.../ByTask/Feeds/QueryMaker.cs | 48 +
.../ByTask/Feeds/RandomFacetSource.cs | 109 ++
.../ByTask/Feeds/ReutersContentSource.cs | 140 +++
.../ByTask/Feeds/ReutersQueryMaker.cs | 126 ++
.../ByTask/Feeds/SimpleQueryMaker.cs | 70 ++
.../Feeds/SimpleSloppyPhraseQueryMaker.cs | 88 ++
.../ByTask/Feeds/SingleDocSource.cs | 77 ++
.../ByTask/Feeds/SortableSingleDocSource.cs | 114 ++
.../ByTask/Feeds/SpatialDocMaker.cs | 249 ++++
.../ByTask/Feeds/SpatialFileQueryMaker.cs | 131 ++
.../ByTask/Feeds/TrecContentSource.cs | 350 ++++++
.../ByTask/Feeds/TrecDocParser.cs | 159 +++
.../ByTask/Feeds/TrecFBISParser.cs | 68 +
.../ByTask/Feeds/TrecFR94Parser.cs | 69 +
.../ByTask/Feeds/TrecFTParser.cs | 58 +
.../ByTask/Feeds/TrecGov2Parser.cs | 57 +
.../ByTask/Feeds/TrecLATimesParser.cs | 75 ++
.../ByTask/Feeds/TrecParserByPath.cs | 34 +
src/Lucene.Net.Benchmark/ByTask/PerfRunData.cs | 490 ++++++++
.../ByTask/Programmatic/Sample.cs | 90 ++
src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs | 108 ++
src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs | 70 ++
.../ByTask/Stats/TaskStats.cs | 237 ++++
.../ByTask/Tasks/AddDocTask.cs | 93 ++
.../ByTask/Tasks/AddFacetedDocTask.cs | 95 ++
.../ByTask/Tasks/AddIndexesTask.cs | 104 ++
.../ByTask/Tasks/AnalyzerFactoryTask.cs | 580 +++++++++
.../ByTask/Tasks/BenchmarkHighlighter.cs | 32 +
.../ByTask/Tasks/ClearStatsTask.cs | 44 +
.../ByTask/Tasks/CloseIndexTask.cs | 67 +
.../ByTask/Tasks/CloseReaderTask.cs | 49 +
.../ByTask/Tasks/CloseTaxonomyIndexTask.cs | 42 +
.../ByTask/Tasks/CloseTaxonomyReaderTask.cs | 47 +
.../ByTask/Tasks/CommitIndexTask.cs | 62 +
.../ByTask/Tasks/CommitTaxonomyIndexTask.cs | 48 +
.../ByTask/Tasks/ConsumeContentSourceTask.cs | 48 +
.../ByTask/Tasks/CreateIndexTask.cs | 225 ++++
.../ByTask/Tasks/CreateTaxonomyIndexTask.cs | 42 +
.../ByTask/Tasks/ForceMergeTask.cs | 61 +
.../ByTask/Tasks/NearRealtimeReaderTask.cs | 132 ++
.../ByTask/Tasks/NewAnalyzerTask.cs | 189 +++
.../ByTask/Tasks/NewCollationAnalyzerTask.cs | 149 +++
.../ByTask/Tasks/NewLocaleTask.cs | 97 ++
.../ByTask/Tasks/NewRoundTask.cs | 44 +
.../ByTask/Tasks/OpenIndexTask.cs | 88 ++
.../ByTask/Tasks/OpenReaderTask.cs | 100 ++
.../ByTask/Tasks/OpenTaxonomyIndexTask.cs | 41 +
.../ByTask/Tasks/OpenTaxonomyReaderTask.cs | 44 +
.../ByTask/Tasks/PerfTask.cs | 380 ++++++
.../ByTask/Tasks/PrintReaderTask.cs | 60 +
.../ByTask/Tasks/ReadTask.cs | 339 +++++
.../ByTask/Tasks/ReadTokensTask.cs | 160 +++
.../ByTask/Tasks/ReopenReaderTask.cs | 45 +
.../ByTask/Tasks/RepAllTask.cs | 83 ++
.../ByTask/Tasks/RepSelectByPrefTask.cs | 81 ++
.../ByTask/Tasks/RepSumByNameRoundTask.cs | 83 ++
.../ByTask/Tasks/RepSumByNameTask.cs | 81 ++
.../ByTask/Tasks/RepSumByPrefRoundTask.cs | 79 ++
.../ByTask/Tasks/RepSumByPrefTask.cs | 91 ++
.../ByTask/Tasks/ReportTask.cs | 189 +++
.../ByTask/Tasks/ResetInputsTask.cs | 43 +
.../ByTask/Tasks/ResetSystemEraseTask.cs | 42 +
.../ByTask/Tasks/ResetSystemSoftTask.cs | 41 +
.../ByTask/Tasks/RollbackIndexTask.cs | 52 +
.../ByTask/Tasks/SearchTask.cs | 60 +
.../ByTask/Tasks/SearchTravRetHighlightTask.cs | 188 +++
.../Tasks/SearchTravRetLoadFieldSelectorTask.cs | 85 ++
.../ByTask/Tasks/SearchTravRetTask.cs | 44 +
.../Tasks/SearchTravRetVectorHighlightTask.cs | 191 +++
.../ByTask/Tasks/SearchTravTask.cs | 87 ++
.../ByTask/Tasks/SearchWithCollectorTask.cs | 99 ++
.../ByTask/Tasks/SearchWithSortTask.cs | 157 +++
.../ByTask/Tasks/SetPropTask.cs | 71 ++
.../ByTask/Tasks/TaskSequence.cs | 662 ++++++++++
.../ByTask/Tasks/UpdateDocTask.cs | 99 ++
.../ByTask/Tasks/WaitForMergesTask.cs | 36 +
.../ByTask/Tasks/WaitTask.cs | 89 ++
.../ByTask/Tasks/WarmTask.cs | 64 +
.../ByTask/Tasks/WriteEnwikiLineDocTask.cs | 72 ++
.../ByTask/Tasks/WriteLineDocTask.cs | 238 ++++
.../ByTask/Utils/Algorithm.cs | 459 +++++++
.../ByTask/Utils/AnalyzerFactory.cs | 156 +++
src/Lucene.Net.Benchmark/ByTask/Utils/Config.cs | 559 +++++++++
.../ByTask/Utils/FileUtils.cs | 46 +
src/Lucene.Net.Benchmark/ByTask/Utils/Format.cs | 109 ++
.../ByTask/Utils/StreamUtils.cs | 132 ++
src/Lucene.Net.Benchmark/Constants.cs | 33 +
.../Lucene.Net.Benchmark.csproj | 214 ++++
.../Lucene.Net.Benchmark.project.json | 15 +
.../Properties/AssemblyInfo.cs | 30 +
src/Lucene.Net.Benchmark/Quality/Judge.cs | 55 +
.../Quality/QualityBenchmark.cs | 159 +++
.../Quality/QualityQuery.cs | 107 ++
.../Quality/QualityQueryParser.cs | 35 +
.../Quality/QualityStats.cs | 339 +++++
.../Quality/Trec/QueryDriver.cs | 93 ++
.../Quality/Trec/Trec1MQReader.cs | 92 ++
.../Quality/Trec/TrecJudge.cs | 186 +++
.../Quality/Trec/TrecTopicsReader.cs | 154 +++
.../Quality/Utils/DocNameExtractor.cs | 89 ++
.../Quality/Utils/QualityQueriesFinder.cs | 152 +++
.../Quality/Utils/SimpleQQParser.cs | 76 ++
.../Quality/Utils/SubmissionReport.cs | 98 ++
.../Utils/ExtractReuters.cs | 167 +++
.../Utils/ExtractWikipedia.cs | 178 +++
src/Lucene.Net.Benchmark/project.json | 53 +
src/Lucene.Net.TestFramework/Util/TestUtil.cs | 22 +-
.../BenchmarkTestCase.cs | 129 ++
.../ByTask/Feeds/DocMakerTest.cs | 193 +++
.../ByTask/Feeds/EnwikiContentSourceTest.cs | 194 +++
.../ByTask/Feeds/LineDocSourceTest.cs | 271 ++++
.../ByTask/Feeds/TestHtmlParser.cs | 164 +++
.../ByTask/Feeds/TrecContentSourceTest.cs | 431 +++++++
.../ByTask/Feeds/trecdocs.zip | Bin 0 -> 2514 bytes
.../ByTask/Tasks/AddIndexesTaskTest.cs | 153 +++
.../ByTask/Tasks/Alt/AltPackageTaskTest.cs | 68 +
.../ByTask/Tasks/Alt/AltTestTask.cs | 35 +
.../ByTask/Tasks/CommitIndexTaskTest.cs | 63 +
.../ByTask/Tasks/CountingHighlighterTestTask.cs | 85 ++
.../ByTask/Tasks/CountingSearchTestTask.cs | 65 +
.../ByTask/Tasks/CreateIndexTaskTest.cs | 129 ++
.../ByTask/Tasks/PerfTaskTest.cs | 81 ++
.../ByTask/Tasks/SearchWithSortTaskTest.cs | 35 +
.../ByTask/Tasks/WriteEnwikiLineDocTaskTest.cs | 121 ++
.../ByTask/Tasks/WriteLineDocTaskTest.cs | 436 +++++++
.../ByTask/TestPerfTasksLogic.cs | 1177 ++++++++++++++++++
.../ByTask/TestPerfTasksParse.cs | 178 +++
.../ByTask/Utils/StreamUtilsTest.cs | 149 +++
.../ByTask/Utils/TestConfig.cs | 37 +
src/Lucene.Net.Tests.Benchmark/ByTask/conf.zip | Bin 0 -> 40878 bytes
.../ByTask/reuters.first20.lines.txt | 20 +
.../test-mapping-ISOLatin1Accent-partial.txt | 30 +
.../Conf/ConfLoader.cs | 28 +
.../Lucene.Net.Tests.Benchmark.csproj | 129 ++
.../Lucene.Net.Tests.Benchmark.project.json | 13 +
.../Properties/AssemblyInfo.cs | 36 +
.../Quality/TestQualityRun.cs | 210 ++++
.../Quality/reuters.578.lines.txt.bz2 | Bin 0 -> 208314 bytes
.../Quality/trecQRels.txt | 723 +++++++++++
.../Quality/trecTopics.txt | 287 +++++
.../Support/TestApiConsistency.cs | 150 +++
.../Support/TestExceptionSerialization.cs | 54 +
src/Lucene.Net.Tests.Benchmark/project.json | 56 +
162 files changed, 22383 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/Lucene.Net.sln
----------------------------------------------------------------------
diff --git a/Lucene.Net.sln b/Lucene.Net.sln
index 5450020..08a00a0 100644
--- a/Lucene.Net.sln
+++ b/Lucene.Net.sln
@@ -110,6 +110,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Analysis.Kuromoj
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Tests.Analysis.Kuromoji", "src\Lucene.Net.Tests.Analysis.Kuromoji\Lucene.Net.Tests.Analysis.Kuromoji.csproj", "{34A2BCE8-1351-43BD-A365-F50E7C0B2C49}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Benchmark", "src\Lucene.Net.Benchmark\Lucene.Net.Benchmark.csproj", "{EDC77CB4-597F-4818-8C83-3C006D12C384}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Tests.Benchmark", "src\Lucene.Net.Tests.Benchmark\Lucene.Net.Tests.Benchmark.csproj", "{9257F543-44E2-4DB6-8B27-A8A354C13E5B}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -1111,6 +1115,54 @@ Global
{34A2BCE8-1351-43BD-A365-F50E7C0B2C49}.Release35|Mixed Platforms.Build.0 = Release|Any CPU
{34A2BCE8-1351-43BD-A365-F50E7C0B2C49}.Release35|x86.ActiveCfg = Release|Any CPU
{34A2BCE8-1351-43BD-A365-F50E7C0B2C49}.Release35|x86.Build.0 = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug|x86.Build.0 = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug35|Any CPU.ActiveCfg = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug35|Any CPU.Build.0 = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug35|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug35|Mixed Platforms.Build.0 = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug35|x86.ActiveCfg = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Debug35|x86.Build.0 = Debug|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release|Any CPU.Build.0 = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release|x86.ActiveCfg = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release|x86.Build.0 = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release35|Any CPU.ActiveCfg = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release35|Any CPU.Build.0 = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release35|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release35|Mixed Platforms.Build.0 = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release35|x86.ActiveCfg = Release|Any CPU
+ {EDC77CB4-597F-4818-8C83-3C006D12C384}.Release35|x86.Build.0 = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug|x86.Build.0 = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug35|Any CPU.ActiveCfg = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug35|Any CPU.Build.0 = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug35|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug35|Mixed Platforms.Build.0 = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug35|x86.ActiveCfg = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Debug35|x86.Build.0 = Debug|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release|x86.ActiveCfg = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release|x86.Build.0 = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release35|Any CPU.ActiveCfg = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release35|Any CPU.Build.0 = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release35|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release35|Mixed Platforms.Build.0 = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release35|x86.ActiveCfg = Release|Any CPU
+ {9257F543-44E2-4DB6-8B27-A8A354C13E5B}.Release35|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Benchmark.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Benchmark.cs b/src/Lucene.Net.Benchmark/ByTask/Benchmark.cs
new file mode 100644
index 0000000..9f3ad70
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Benchmark.cs
@@ -0,0 +1,170 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Run the benchmark algorithm.
+ /// </summary>
+ /// <remarks>
+ /// <list type="number">
+ /// <item><description>Read algorithm.</description></item>
+ /// <item><description>Run the algorithm.</description></item>
+ /// </list>
+ /// <para/>
+ /// Things to be added/fixed in "Benchmarking by tasks":
+ /// <list type="number">
+ /// <item><description>TODO - report into Excel and/or graphed view.</description></item>
+ /// <item><description>TODO - perf comparison between Lucene releases over the years.</description></item>
+ /// <item><description>TODO - perf report adequate to include in Lucene nightly build site? (so we can easily track performance changes.)</description></item>
+ /// <item><description>TODO - add overall time control for repeated execution (vs. current by-count only).</description></item>
+ /// <item><description>TODO - query maker that is based on index statistics.</description></item>
+ /// </list>
+ /// </remarks>
+ public class Benchmark
+ {
+ private PerfRunData runData;
+ private Algorithm algorithm;
+ private bool executed;
+
+ public Benchmark(TextReader algReader)
+ {
+ // prepare run data
+ try
+ {
+ runData = new PerfRunData(new Config(algReader));
+ }
+ catch (Exception e)
+ {
+ //e.printStackTrace();
+ throw new Exception("Error: cannot init PerfRunData!", e);
+ }
+
+ // parse algorithm
+ try
+ {
+ algorithm = new Algorithm(runData);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("Error: cannot understand algorithm!", e);
+ }
+ }
+
+ /// <summary>
+ /// Execute this benchmark.
+ /// </summary>
+ public virtual void Execute()
+ {
+ lock (this)
+ {
+ if (executed)
+ {
+ throw new InvalidOperationException("Benchmark was already executed");
+ }
+ executed = true;
+ runData.SetStartTimeMillis();
+ algorithm.Execute();
+ }
+ }
+
+ /// <summary>
+ /// Run the benchmark algorithm.
+ /// </summary>
+ /// <param name="args">Benchmark config and algorithm files.</param>
+ public static void Main(string[] args)
+ {
+ Exec(args);
+ }
+
+ /// <summary>
+ /// Utility: execute benchmark from command line.
+ /// </summary>
+ /// <param name="args">Single argument is expected: algorithm-file.</param>
+ public static void Exec(string[] args)
+ {
+ // verify command line args
+ if (args.Length < 1)
+ {
+ SystemConsole.WriteLine("Usage: java Benchmark <algorithm file>");
+ Environment.Exit(1);
+ }
+
+ // verify input files
+ FileInfo algFile = new FileInfo(args[0]);
+ if (!algFile.Exists /*|| !algFile.isFile() ||!algFile.canRead()*/ )
+ {
+ SystemConsole.WriteLine("cannot find/read algorithm file: " + algFile.FullName);
+ Environment.Exit(1);
+ }
+
+ SystemConsole.WriteLine("Running algorithm from: " + algFile.FullName);
+
+ Benchmark benchmark = null;
+ try
+ {
+ benchmark = new Benchmark(IOUtils.GetDecodingReader(algFile, Encoding.UTF8));
+ }
+ catch (Exception e)
+ {
+ SystemConsole.WriteLine(e.ToString());
+ Environment.Exit(1);
+ }
+
+ SystemConsole.WriteLine("------------> algorithm:");
+ SystemConsole.WriteLine(benchmark.Algorithm.ToString());
+
+ // execute
+ try
+ {
+ benchmark.Execute();
+ }
+ catch (Exception e)
+ {
+ SystemConsole.WriteLine("Error: cannot execute the algorithm! " + e.Message);
+ SystemConsole.WriteLine(e.StackTrace);
+ }
+
+ SystemConsole.WriteLine("####################");
+ SystemConsole.WriteLine("### D O N E !!! ###");
+ SystemConsole.WriteLine("####################");
+ }
+
+ /// <summary>
+ /// Returns the algorithm.
+ /// </summary>
+ public virtual Algorithm Algorithm
+ {
+ get { return algorithm; }
+ }
+
+ /// <summary>
+ /// Returns the runData.
+ /// </summary>
+ public virtual PerfRunData RunData
+ {
+ get { return runData; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/AbstractQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/AbstractQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/AbstractQueryMaker.cs
new file mode 100644
index 0000000..fb6a2bf
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/AbstractQueryMaker.cs
@@ -0,0 +1,85 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Search;
+using System;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Abstract base query maker.
+ /// Each query maker should just implement the <see cref="PrepareQueries()"/> method.
+ /// </summary>
+ public abstract class AbstractQueryMaker : IQueryMaker
+ {
+ protected int m_qnum = 0;
+ protected Query[] m_queries;
+ protected Config m_config;
+
+ public virtual void ResetInputs()
+ {
+ m_qnum = 0;
+ }
+
+ protected abstract Query[] PrepareQueries();
+
+ public virtual void SetConfig(Config config)
+ {
+ this.m_config = config;
+ m_queries = PrepareQueries();
+ }
+
+ public virtual string PrintQueries()
+ {
+ string newline = Environment.NewLine;
+ StringBuilder sb = new StringBuilder();
+ if (m_queries != null)
+ {
+ for (int i = 0; i < m_queries.Length; i++)
+ {
+ sb.Append(i + ". " + m_queries[i].GetType().Name + " - " + m_queries[i].ToString());
+ sb.Append(newline);
+ }
+ }
+ return sb.ToString();
+ }
+
+ public virtual Query MakeQuery()
+ {
+ return m_queries[NextQnum()];
+ }
+
+ // return next qnum
+ protected virtual int NextQnum()
+ {
+ lock (this)
+ {
+ int res = m_qnum;
+ m_qnum = (m_qnum + 1) % m_queries.Length;
+ return res;
+ }
+ }
+
+ /// <seealso cref="IQueryMaker.MakeQuery(int)"/>
+ public virtual Query MakeQuery(int size)
+ {
+ throw new Exception(this + ".MakeQuery(int size) is not supported!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentItemsSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentItemsSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentItemsSource.cs
new file mode 100644
index 0000000..c0f06ef
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentItemsSource.cs
@@ -0,0 +1,227 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Base class for source of data for benchmarking.
+ /// </summary>
+ /// <remarks>
+ /// Keeps track of various statistics, such as how many data items were generated,
+ /// size in bytes etc.
+ /// <para/>
+ /// Supports the following configuration parameters:
+ /// <list type="bullet">
+ /// <item><term>content.source.forever</term><description>specifies whether to generate items forever (<b>default=true</b>).</description></item>
+ /// <item><term>content.source.verbose</term><description>specifies whether messages should be output by the content source (<b>default=false</b>).</description></item>
+ /// <item><term>content.source.encoding</term><description>
+ /// specifies which encoding to use when
+ /// reading the files of that content source. Certain implementations may define
+ /// a default value if this parameter is not specified. (<b>default=null</b>).
+ /// </description></item>
+ /// <item><term>content.source.log.step</term><description>
+ /// specifies for how many items a
+ /// message should be logged. If set to 0 it means no logging should occur.
+ /// <b>NOTE:</b> if verbose is set to false, logging should not occur even if
+ /// logStep is not 0 (<b>default=0</b>).
+ /// </description></item>
+ /// </list>
+ /// </remarks>
+ public abstract class ContentItemsSource : IDisposable
+ {
+ private long bytesCount;
+ private long totalBytesCount;
+ private int itemCount;
+ private int totalItemCount;
+ private Config config;
+
+ private int lastPrintedNumUniqueTexts = 0;
+ private long lastPrintedNumUniqueBytes = 0;
+ private int printNum = 0;
+
+ protected bool m_forever;
+ protected int m_logStep;
+ protected bool m_verbose;
+ protected Encoding m_encoding;
+
+ /// <summary>update count of bytes generated by this source</summary>
+ protected void AddBytes(long numBytes)
+ {
+ lock (this)
+ {
+ bytesCount += numBytes;
+ totalBytesCount += numBytes;
+ }
+ }
+
+ /// <summary>update count of items generated by this source</summary>
+ protected void AddItem()
+ {
+ lock (this)
+ {
+ ++itemCount;
+ ++totalItemCount;
+ }
+ }
+
+ /// <summary>
+ /// A convenience method for collecting all the files of a content source from
+ /// a given directory. The collected <see cref="FileInfo"/> instances are stored in the
+ /// given <paramref name="files"/>.
+ /// </summary>
+ protected void CollectFiles(DirectoryInfo dir, IList<FileInfo> files)
+ {
+ CollectFilesImpl(dir, files);
+ files.Sort(new FileNameComparer());
+ }
+
+ private void CollectFilesImpl(DirectoryInfo dir, IList<FileInfo> files)
+ {
+ foreach (var sub in dir.EnumerateDirectories())
+ {
+ CollectFilesImpl(sub, files);
+ }
+
+ files.AddRange(dir.GetFiles());
+ }
+
+ private class FileNameComparer : IComparer<FileInfo>
+ {
+ public int Compare(FileInfo x, FileInfo y)
+ {
+ return x.FullName.CompareToOrdinal(y.FullName);
+ }
+ }
+
+ /// <summary>
+ /// Returns <c>true</c> whether it's time to log a message (depending on verbose and
+ /// the number of items generated).
+ /// </summary>
+ /// <returns></returns>
+ protected bool ShouldLog()
+ {
+ return m_verbose && m_logStep > 0 && itemCount % m_logStep == 0;
+ }
+
+ /// <summary>Called when reading from this content source is no longer required.</summary>
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ /// <summary>Called when reading from this content source is no longer required.</summary>
+ protected abstract void Dispose(bool disposing);
+
+
+ /// <summary>Returns the number of bytes generated since last reset.</summary>
+ public long BytesCount { get { return bytesCount; } }
+
+ /// <summary>Returns the number of generated items since last reset.</summary>
+ public int ItemsCount { get { return itemCount; } }
+
+ public Config Config { get { return config; } }
+
+ /// <summary>Returns the total number of bytes that were generated by this source.</summary>
+ public long TotalBytesCount { get { return totalBytesCount; } }
+
+ /// <summary>Returns the total number of generated items.</summary>
+ public int TotalItemsCount { get { return totalItemCount; } }
+
+ /// <summary>
+ /// Resets the input for this content source, so that the test would behave as
+ /// if it was just started, input-wise.
+ /// <para/>
+ /// <b>NOTE:</b> the default implementation resets the number of bytes and
+ /// items generated since the last reset, so it's important to call
+ /// <c>base.ResetInputs()</c> in case you override this method.
+ /// </summary>
+ public virtual void ResetInputs()
+ {
+ bytesCount = 0;
+ itemCount = 0;
+ }
+
+ /// <summary>
+ /// Sets the <see cref="Utils.Config"/> for this content source. If you override this
+ /// method, you must call <c>base.SetConfig(config)</c>.
+ /// </summary>
+ /// <param name="config"></param>
+ public virtual void SetConfig(Config config)
+ {
+ this.config = config;
+ m_forever = config.Get("content.source.forever", true);
+ m_logStep = config.Get("content.source.log.step", 0);
+ m_verbose = config.Get("content.source.verbose", false);
+ string encodingStr = config.Get("content.source.encoding", null);
+ if (!string.IsNullOrWhiteSpace(encodingStr))
+ {
+ m_encoding = Encoding.GetEncoding(encodingStr);
+ }
+ else
+ {
+ m_encoding = Encoding.GetEncoding(0); // Default system encoding
+ }
+ }
+
+ public virtual void PrintStatistics(string itemsName)
+ {
+ if (!m_verbose)
+ {
+ return;
+ }
+ bool print = false;
+ string col = " ";
+ StringBuilder sb = new StringBuilder();
+ string newline = Environment.NewLine;
+ sb.Append("------------> ").Append(GetType().GetTypeInfo().Name).Append(" statistics (").Append(printNum).Append("): ").Append(newline);
+ int nut = TotalItemsCount;
+ if (nut > lastPrintedNumUniqueTexts)
+ {
+ print = true;
+ sb.Append("total count of " + itemsName + ": ").Append(Formatter.Format(0, nut, col)).Append(newline);
+ lastPrintedNumUniqueTexts = nut;
+ }
+ long nub = TotalBytesCount;
+ if (nub > lastPrintedNumUniqueBytes)
+ {
+ print = true;
+ sb.Append("total bytes of " + itemsName + ": ").Append(Formatter.Format(0, nub, col)).Append(newline);
+ lastPrintedNumUniqueBytes = nub;
+ }
+ if (ItemsCount > 0)
+ {
+ print = true;
+ sb.Append("num " + itemsName + " added since last inputs reset: ").Append(Formatter.Format(0, ItemsCount, col)).Append(newline);
+ sb.Append("total bytes added for " + itemsName + " since last inputs reset: ").Append(Formatter.Format(0, BytesCount, col)).Append(newline);
+ }
+ if (print)
+ {
+ SystemConsole.WriteLine(sb.Append(newline).ToString());
+ printNum++;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentSource.cs
new file mode 100644
index 0000000..a3c39cb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/ContentSource.cs
@@ -0,0 +1,38 @@
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Represents content from a specified source, such as TREC, Reuters etc. A
+ /// <see cref="ContentSource"/> is responsible for creating <see cref="DocData"/> objects for
+ /// its documents to be consumed by <see cref="DocMaker"/>. It also keeps track
+ /// of various statistics, such as how many documents were generated, size in
+ /// bytes etc.
+ /// <para/>
+ /// For supported configuration parameters see <see cref="ContentItemsSource"/>.
+ /// </summary>
+ public abstract class ContentSource : ContentItemsSource
+ {
+ /// <summary>
+ /// Returns the next <see cref="DocData"/> from the content source.
+ /// Implementations must account for multi-threading, as multiple threads
+ /// can call this method simultaneously.
+ /// </summary>
+ public abstract DocData GetNextDocData(DocData docData);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
new file mode 100644
index 0000000..0903754
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/DemoHTMLParser.cs
@@ -0,0 +1,259 @@
+// LUCENENET TODO: Use HTML Agility pack instead of SAX ?
+
+using Lucene.Net.Support;
+using Sax.Net;
+using Sax.Net.Helpers;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simple HTML Parser extracting title, meta tags, and body text
+ /// that is based on <a href="http://nekohtml.sourceforge.net/">NekoHTML</a>.
+ /// </summary>
+ public class DemoHTMLParser : IHTMLParser
+ {
+ /// <summary>The actual parser to read HTML documents.</summary>
+ public sealed class Parser
+ {
+ private readonly IDictionary<string, string> metaTags = new Dictionary<string, string>();
+ private readonly string title, body;
+
+ // LUCENENET specific - expose field through property
+ public IDictionary<string, string> MetaTags
+ {
+ get { return metaTags; }
+ }
+
+ // LUCENENET specific - expose field through property
+ public string Title
+ {
+ get { return title; }
+ }
+
+ // LUCENENET specific - expose field through property
+ public string Body
+ {
+ get { return body; }
+ }
+
+ public Parser(TextReader reader)
+ : this(new InputSource(reader))
+ {
+ }
+
+ public Parser(InputSource source)
+ {
+ TagSoup.Net.Parser parser = new TagSoup.Net.Parser();
+
+ parser.SetFeature(TagSoup.Net.Parser.NAMESPACES_FEATURE, true);
+
+ StringBuilder title = new StringBuilder(), body = new StringBuilder();
+ DefaultHandler handler = new DefaultHandlerAnonymousHelper(this, title, body);
+
+ parser.ContentHandler = handler;
+ parser.ErrorHandler = handler;
+ parser.Parse(source);
+
+ // the javacc-based parser trimmed title (which should be done for HTML in all cases):
+ this.title = title.ToString().Trim();
+
+ // assign body text
+ this.body = body.ToString();
+ }
+
+ private class DefaultHandlerAnonymousHelper : DefaultHandler
+ {
+ private int inBODY = 0, inHEAD = 0, inTITLE = 0, suppressed = 0;
+
+ private readonly Parser outerInstance;
+ private readonly StringBuilder title;
+ private readonly StringBuilder body;
+
+ public DefaultHandlerAnonymousHelper(Parser outerInstance, StringBuilder title, StringBuilder body)
+ {
+ this.outerInstance = outerInstance;
+ this.title = title;
+ this.body = body;
+ }
+
+ public override void StartElement(string uri, string localName, string qName, IAttributes atts)
+ {
+ if (inHEAD > 0)
+ {
+ if ("title".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ inTITLE++;
+ }
+ else
+ {
+ if ("meta".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ string name = atts.GetValue("name");
+ if (name == null)
+ {
+ name = atts.GetValue("http-equiv");
+ }
+ string val = atts.GetValue("content");
+ if (name != null && val != null)
+ {
+ outerInstance.metaTags[name.ToLowerInvariant()] = val;
+ }
+ }
+ }
+ }
+ else if (inBODY > 0)
+ {
+ if (SUPPRESS_ELEMENTS.Contains(localName))
+ {
+ suppressed++;
+ }
+ else if ("img".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ // the original javacc-based parser preserved <IMG alt="..."/>
+ // attribute as body text in [] parenthesis:
+ string alt = atts.GetValue("alt");
+ if (alt != null)
+ {
+ body.Append('[').Append(alt).Append(']');
+ }
+ }
+ }
+ else if ("body".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ inBODY++;
+ }
+ else if ("head".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ inHEAD++;
+ }
+ else if ("frameset".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ throw new SAXException("This parser does not support HTML framesets.");
+ }
+ }
+
+ public override void EndElement(string uri, string localName, string qName)
+ {
+ if (inBODY > 0)
+ {
+ if ("body".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ inBODY--;
+ }
+ else if (ENDLINE_ELEMENTS.Contains(localName))
+ {
+ body.Append('\n');
+ }
+ else if (SUPPRESS_ELEMENTS.Contains(localName))
+ {
+ suppressed--;
+ }
+ }
+ else if (inHEAD > 0)
+ {
+ if ("head".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ inHEAD--;
+ }
+ else if (inTITLE > 0 && "title".Equals(localName, StringComparison.OrdinalIgnoreCase))
+ {
+ inTITLE--;
+ }
+ }
+ }
+
+ public override void Characters(char[] ch, int start, int length)
+ {
+ if (inBODY > 0 && suppressed == 0)
+ {
+ body.Append(ch, start, length);
+ }
+ else if (inTITLE > 0)
+ {
+ title.Append(ch, start, length);
+ }
+ }
+
+ public override InputSource ResolveEntity(string publicId, string systemId)
+ {
+ // disable network access caused by DTDs
+ return new InputSource(new StringReader(""));
+ }
+ }
+
+ private static ISet<string> CreateElementNameSet(params string[] names)
+ {
+ return Collections.UnmodifiableSet(new HashSet<string>(names));
+ }
+
+ /// <summary>HTML elements that cause a line break (they are block-elements).</summary>
+ internal static readonly ISet<string> ENDLINE_ELEMENTS = CreateElementNameSet(
+ "p", "h1", "h2", "h3", "h4", "h5", "h6", "div", "ul", "ol", "dl",
+ "pre", "hr", "blockquote", "address", "fieldset", "table", "form",
+ "noscript", "li", "dt", "dd", "noframes", "br", "tr", "select", "option"
+ );
+
+ /// <summary>HTML elements with contents that are ignored.</summary>
+ internal static readonly ISet<string> SUPPRESS_ELEMENTS = CreateElementNameSet(
+ "style", "script"
+ );
+ }
+ public virtual DocData Parse(DocData docData, string name, DateTime? date, TextReader reader, TrecContentSource trecSrc)
+ {
+ try
+ {
+ return Parse(docData, name, date, new InputSource(reader), trecSrc);
+ }
+ catch (SAXException saxe)
+ {
+ throw new IOException("SAX exception occurred while parsing HTML document.", saxe);
+ }
+ }
+
+ public virtual DocData Parse(DocData docData, string name, DateTime? date, InputSource source, TrecContentSource trecSrc)
+ {
+ Parser p = new Parser(source);
+
+ // properties
+ IDictionary<string, string> props = p.MetaTags;
+ string dateStr;
+ if (props.TryGetValue("date", out dateStr) && dateStr != null)
+ {
+ DateTime? newDate = trecSrc.ParseDate(dateStr);
+ if (newDate != null)
+ {
+ date = newDate;
+ }
+ }
+
+ docData.Clear();
+ docData.Name = name;
+ docData.Body = p.Body;
+ docData.Title = p.Title;
+ docData.Props = props;
+ docData.SetDate(date);
+ return docData;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/DirContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/DirContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/DirContentSource.cs
new file mode 100644
index 0000000..c14d578
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/DirContentSource.cs
@@ -0,0 +1,259 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+// LUCENENET TODO: This had to be refactored significantly. We need tests to confirm it works.
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="ContentSource"/> using the Dir collection for its input. Supports
+ /// the following configuration parameters (on top of <see cref="ContentSource"/>):
+ /// <list type="bullet">
+ /// <item><term>work.dir</term><description>specifies the working directory. Required if "docs.dir" denotes a relative path (<b>default=work</b>).</description></item>
+ /// <item><term>docs.dir</term><description>specifies the directory the Dir collection. Can be set to a relative path if "work.dir" is also specified (<b>default=dir-out</b>).</description></item>
+ /// </list>
+ /// </summary>
+ public class DirContentSource : ContentSource
+ {
+ /// <summary>
+ /// Iterator over the files in the directory.
+ /// </summary>
+ public class Iterator : IEnumerator<FileInfo>
+ {
+
+ private class Comparer : IComparer<FileInfo>
+ {
+ public int Compare(FileInfo a, FileInfo b)
+ {
+ string a2 = a.ToString();
+ string b2 = b.ToString();
+ int diff = a2.Length - b2.Length;
+
+ if (diff > 0)
+ {
+ while (diff-- > 0)
+ {
+ b2 = "0" + b2;
+ }
+ }
+ else if (diff < 0)
+ {
+ diff = -diff;
+ while (diff-- > 0)
+ {
+ a2 = "0" + a2;
+ }
+ }
+
+ /* note it's reversed because we're going to push,
+ which reverses again */
+ return b2.CompareToOrdinal(a2);
+ }
+ }
+
+ internal int count = 0;
+
+ internal Stack<FileInfo> stack = new Stack<FileInfo>();
+
+ /* this seems silly ... there must be a better way ...
+ not that this is good, but can it matter? */
+
+ private Comparer c = new Comparer();
+
+ private FileInfo current;
+
+ public Iterator(DirectoryInfo f)
+ {
+ Push(f);
+ }
+
+ internal void Push(DirectoryInfo f)
+ {
+ foreach (var dir in f.GetDirectories())
+ {
+ Push(dir);
+ }
+
+ Push(f.GetFiles("*.txt"));
+ }
+
+ internal void Push(FileInfo[] files)
+ {
+ Array.Sort(files, c);
+ for (int i = 0; i < files.Length; i++)
+ {
+ // System.err.println("push " + files[i]);
+ stack.Push(files[i]);
+ }
+ }
+
+ public virtual int Count
+ {
+ get { return count; }
+ }
+
+ public virtual bool MoveNext()
+ {
+ if (stack.Count == 0)
+ {
+ current = null;
+ return false;
+ }
+ count++;
+ current = stack.Pop();
+ // System.err.println("pop " + object);
+ return true;
+ }
+
+ public virtual FileInfo Current
+ {
+ get { return current; }
+ }
+
+ object IEnumerator.Current
+ {
+ get { return current; }
+ }
+
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ }
+
+ public virtual void Reset()
+ {
+ }
+ }
+
+ private DirectoryInfo dataDir = null;
+ private int iteration = 0;
+ private Iterator inputFiles = null;
+
+ private DateTime? ParseDate(string dateStr)
+ {
+ DateTime temp;
+ if (DateTime.TryParseExact(dateStr, "dd-MMM-yyyy hh:mm:ss.fff", CultureInfo.InvariantCulture, DateTimeStyles.None, out temp))
+ {
+ return temp;
+ }
+ else if (DateTime.TryParse(dateStr, CultureInfo.InvariantCulture, DateTimeStyles.None, out temp))
+ {
+ return temp;
+ }
+
+ return null;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ inputFiles = null;
+ }
+ }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ FileInfo f = null;
+ string name = null;
+ lock (this)
+ {
+ if (!inputFiles.MoveNext())
+ {
+ // exhausted files, start a new round, unless forever set to false.
+ if (!m_forever)
+ {
+ throw new NoMoreDataException();
+ }
+ inputFiles = new Iterator(dataDir);
+ iteration++;
+ }
+ f = inputFiles.Current;
+ // System.err.println(f);
+ name = f.FullName + "_" + iteration;
+ }
+
+ string line = null;
+ string dateStr;
+ string title;
+ StringBuilder bodyBuf = new StringBuilder(1024);
+
+ using (TextReader reader = new StreamReader(new FileStream(f.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8))
+ {
+ //First line is the date, 3rd is the title, rest is body
+ dateStr = reader.ReadLine();
+ reader.ReadLine();//skip an empty line
+ title = reader.ReadLine();
+ reader.ReadLine();//skip an empty line
+ while ((line = reader.ReadLine()) != null)
+ {
+ bodyBuf.Append(line).Append(' ');
+ }
+ }
+ AddBytes(f.Length);
+
+ DateTime? date = ParseDate(dateStr);
+
+ docData.Clear();
+ docData.Name = name;
+ docData.Body = bodyBuf.ToString();
+ docData.Title = title;
+ docData.SetDate(date);
+ return docData;
+ }
+
+ public override void ResetInputs()
+ {
+ lock (this)
+ {
+ base.ResetInputs();
+ inputFiles = new Iterator(dataDir);
+ iteration = 0;
+ }
+ }
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+
+ DirectoryInfo workDir = new DirectoryInfo(config.Get("work.dir", "work"));
+ string d = config.Get("docs.dir", "dir-out");
+ dataDir = new DirectoryInfo(d);
+
+ inputFiles = new Iterator(dataDir);
+
+ if (inputFiles == null)
+ {
+ throw new Exception("No txt files in dataDir: " + dataDir.FullName);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/DocData.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/DocData.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/DocData.cs
new file mode 100644
index 0000000..9e68a4e
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/DocData.cs
@@ -0,0 +1,73 @@
+using Lucene.Net.Documents;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Output of parsing (e.g. HTML parsing) of an input document.
+ /// </summary>
+ public class DocData
+ {
+ public string Name { get; set; }
+ public string Body { get; set; }
+ public string Title { get; set; }
+ private string date;
+ public int ID { get; set; }
+ public IDictionary<string, string> Props { get; set; }
+
+ public void Clear()
+ {
+ Name = null;
+ Body = null;
+ Title = null;
+ date = null;
+ Props = null;
+ ID = -1;
+ }
+
+ /// <summary>
+ /// Gets the date. If the ctor with <see cref="DateTime"/> was called, then the string
+ /// returned is the output of <see cref="DateTools.DateToString(DateTime, DateTools.Resolution)"/>.
+ /// Otherwise it's the string passed to the other ctor.
+ /// </summary>
+ public virtual string Date
+ {
+ get { return date; }
+ }
+
+ public virtual void SetDate(DateTime? date)
+ {
+ if (date.HasValue)
+ {
+ SetDate(DateTools.DateToString(date.Value, DateTools.Resolution.SECOND));
+ }
+ else
+ {
+ this.date = null;
+ }
+ }
+
+ public virtual void SetDate(string date)
+ {
+ this.date = date;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/DocMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/DocMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/DocMaker.cs
new file mode 100644
index 0000000..8ff3e7b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/DocMaker.cs
@@ -0,0 +1,511 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Creates <see cref="Document"/> objects. Uses a <see cref="ContentSource"/> to generate
+ /// <see cref="DocData"/> objects.
+ /// </summary>
+ /// <remarks>
+ /// Supports the following parameters:
+ /// <list type="bullet">
+ /// <item><term>content.source</term><description>specifies the <see cref="ContentSource"/> class to use (default <b>SingleDocSource</b>).</description></item>
+ /// <item><term>doc.stored</term><description>specifies whether fields should be stored (default <b>false</b>).</description></item>
+ /// <item><term>doc.body.stored</term><description>specifies whether the body field should be stored (default = <b>doc.stored</b>).</description></item>
+ /// <item><term>doc.tokenized</term><description>specifies whether fields should be tokenized (default <b>true</b>).</description></item>
+ /// <item><term>doc.body.tokenized</term><description>specifies whether the body field should be tokenized (default = <b>doc.tokenized</b>).</description></item>
+ /// <item><term>doc.tokenized.norms</term><description>specifies whether norms should be stored in the index or not. (default <b>false</b>).</description></item>
+ /// <item><term>doc.body.tokenized.norms</term><description>
+ /// specifies whether norms should be stored in the index for the body field.
+ /// This can be set to true, while <c>doc.tokenized.norms</c> is set to false, to allow norms storing just
+ /// for the body field. (default <b>true</b>).
+ /// </description></item>
+ /// <item><term>doc.term.vector</term><description>specifies whether term vectors should be stored for fields (default <b>false</b>).</description></item>
+ /// <item><term>doc.term.vector.positions</term><description>specifies whether term vectors should be stored with positions (default <b>false</b>).</description></item>
+ /// <item><term>doc.term.vector.offsets</term><description>specifies whether term vectors should be stored with offsets (default <b>false</b>).</description></item>
+ /// <item><term>doc.store.body.bytes</term><description>specifies whether to store the raw bytes of the document's content in the document (default <b>false</b>).</description></item>
+ /// <item><term>doc.reuse.fields</term><description>specifies whether <see cref="Field"/> and <see cref="Document"/> objects should be reused (default <b>true</b>).</description></item>
+ /// <item><term>doc.index.props</term><description>specifies whether the properties returned by</description></item>
+ /// <item><term>doc.random.id.limit</term><description>
+ /// if specified, docs will be assigned random
+ /// IDs from 0 to this limit. This is useful with UpdateDoc
+ /// for testing performance of <see cref="Index.IndexWriter.UpdateDocument(Index.Term, IEnumerable{Index.IIndexableField})"/>.
+ /// <see cref="DocData.Props"/> will be indexed. (default <b>false</b>).
+ /// </description></item>
+ /// </list>
+ /// </remarks>
+ public class DocMaker : IDisposable
+ {
+ private class LeftOver
+ {
+ public DocData DocData { get; set; }
+ public int Count { get; set; }
+ }
+
+ private Random r;
+ private int updateDocIDLimit;
+
+ /// <summary>
+ /// Document state, supports reuse of field instances
+ /// across documents (see <c>reuseFields</c> parameter).
+ /// </summary>
+ protected class DocState
+ {
+ private readonly IDictionary<string, Field> fields;
+ private readonly IDictionary<string, Field> numericFields;
+ private readonly bool reuseFields;
+ internal readonly Document doc;
+ internal DocData docData = new DocData();
+
+ public DocState(bool reuseFields, FieldType ft, FieldType bodyFt)
+ {
+
+ this.reuseFields = reuseFields;
+
+ if (reuseFields)
+ {
+ fields = new Dictionary<string, Field>();
+ numericFields = new Dictionary<string, Field>();
+
+ // Initialize the map with the default fields.
+ fields[BODY_FIELD] = new Field(BODY_FIELD, "", bodyFt);
+ fields[TITLE_FIELD] = new Field(TITLE_FIELD, "", ft);
+ fields[DATE_FIELD] = new Field(DATE_FIELD, "", ft);
+ fields[ID_FIELD] = new StringField(ID_FIELD, "", Field.Store.YES);
+ fields[NAME_FIELD] = new Field(NAME_FIELD, "", ft);
+
+ numericFields[DATE_MSEC_FIELD] = new Int64Field(DATE_MSEC_FIELD, 0L, Field.Store.NO);
+ numericFields[TIME_SEC_FIELD] = new Int32Field(TIME_SEC_FIELD, 0, Field.Store.NO);
+
+ doc = new Document();
+ }
+ else
+ {
+ numericFields = null;
+ fields = null;
+ doc = null;
+ }
+ }
+
+ /// <summary>
+ /// Returns a field corresponding to the field name. If
+ /// <c>reuseFields</c> was set to <c>true</c>, then it attempts to reuse a
+ /// <see cref="Field"/> instance. If such a field does not exist, it creates a new one.
+ /// </summary>
+ internal Field GetField(string name, FieldType ft)
+ {
+ if (!reuseFields)
+ {
+ return new Field(name, "", ft);
+ }
+
+ Field f;
+ if (!fields.TryGetValue(name, out f) || f == null)
+ {
+ f = new Field(name, "", ft);
+ fields[name] = f;
+ }
+ return f;
+ }
+
+ internal Field GetNumericField(string name, NumericType type)
+ {
+ Field f;
+ if (reuseFields)
+ {
+ numericFields.TryGetValue(name, out f);
+ }
+ else
+ {
+ f = null;
+ }
+
+ if (f == null)
+ {
+ switch (type)
+ {
+ case NumericType.INT32:
+ f = new Int32Field(name, 0, Field.Store.NO);
+ break;
+ case NumericType.INT64:
+ f = new Int64Field(name, 0L, Field.Store.NO);
+ break;
+ case NumericType.SINGLE:
+ f = new SingleField(name, 0.0F, Field.Store.NO);
+ break;
+ case NumericType.DOUBLE:
+ f = new DoubleField(name, 0.0, Field.Store.NO);
+ break;
+ default:
+ throw new InvalidOperationException("Cannot get here");
+ }
+ if (reuseFields)
+ {
+ numericFields[name] = f;
+ }
+ }
+ return f;
+ }
+ }
+
+ private bool storeBytes = false;
+
+ // LUCENENET specific: DateUtil not used
+
+ // leftovers are thread local, because it is unsafe to share residues between threads
+ private ThreadLocal<LeftOver> leftovr = new ThreadLocal<LeftOver>();
+ private ThreadLocal<DocState> docState = new ThreadLocal<DocState>();
+
+ public static readonly string BODY_FIELD = "body";
+ public static readonly string TITLE_FIELD = "doctitle";
+ public static readonly string DATE_FIELD = "docdate";
+ public static readonly string DATE_MSEC_FIELD = "docdatenum";
+ public static readonly string TIME_SEC_FIELD = "doctimesecnum";
+ public static readonly string ID_FIELD = "docid";
+ public static readonly string BYTES_FIELD = "bytes";
+ public static readonly string NAME_FIELD = "docname";
+
+ protected Config m_config;
+
+ protected FieldType m_valType;
+ protected FieldType m_bodyValType;
+
+ protected ContentSource m_source;
+ protected bool m_reuseFields;
+ protected bool m_indexProperties;
+
+ private readonly AtomicInt32 numDocsCreated = new AtomicInt32();
+
+ public DocMaker()
+ {
+ }
+
+ // create a doc
+ // use only part of the body, modify it to keep the rest (or use all if size==0).
+ // reset the docdata properties so they are not added more than once.
+ private Document CreateDocument(DocData docData, int size, int cnt)
+ {
+
+ DocState ds = GetDocState();
+ Document doc = m_reuseFields ? ds.doc : new Document();
+ doc.Fields.Clear();
+
+ // Set ID_FIELD
+ FieldType ft = new FieldType(m_valType);
+ ft.IsIndexed = true;
+
+ Field idField = ds.GetField(ID_FIELD, ft);
+ int id;
+ if (r != null)
+ {
+ id = r.Next(updateDocIDLimit);
+ }
+ else
+ {
+ id = docData.ID;
+ if (id == -1)
+ {
+ id = numDocsCreated.GetAndIncrement();
+ }
+ }
+ idField.SetStringValue(Convert.ToString(id, CultureInfo.InvariantCulture));
+ doc.Add(idField);
+
+ // Set NAME_FIELD
+ string name = docData.Name;
+ if (name == null) name = "";
+ name = cnt < 0 ? name : name + "_" + cnt;
+ Field nameField = ds.GetField(NAME_FIELD, m_valType);
+ nameField.SetStringValue(name);
+ doc.Add(nameField);
+
+ // Set DATE_FIELD
+ DateTime? date = null;
+ string dateString = docData.Date;
+ if (dateString != null)
+ {
+ // LUCENENET: TryParseExact needs a non-nullable DateTime to work.
+ DateTime temp;
+ if (DateTime.TryParseExact(dateString, new string[] {
+ // Original format from Java
+ "dd-MMM-yyyy HH:mm:ss",
+ // Actual format from the test files...
+ "yyyyMMddHHmmss"
+ }, CultureInfo.InvariantCulture, DateTimeStyles.None, out temp))
+ {
+ date = temp;
+ }
+ // LUCENENET: Hail Mary in case the formats above are not adequate
+ else if (DateTime.TryParse(dateString, CultureInfo.InvariantCulture, DateTimeStyles.None, out temp))
+ {
+ date = temp;
+ }
+ }
+ else
+ {
+ dateString = "";
+ }
+ Field dateStringField = ds.GetField(DATE_FIELD, m_valType);
+ dateStringField.SetStringValue(dateString);
+ doc.Add(dateStringField);
+
+ if (date == null)
+ {
+ // just set to right now
+ date = DateTime.Now;
+ }
+
+ Field dateField = ds.GetNumericField(DATE_MSEC_FIELD, NumericType.INT64);
+ dateField.SetInt64Value(date.Value.Ticks);
+ doc.Add(dateField);
+
+ //util.cal.setTime(date);
+ //int sec = util.cal.get(Calendar.HOUR_OF_DAY) * 3600 + util.cal.get(Calendar.MINUTE) * 60 + util.cal.get(Calendar.SECOND);
+ int sec = Convert.ToInt32(date.Value.ToUniversalTime().TimeOfDay.TotalSeconds);
+
+ Field timeSecField = ds.GetNumericField(TIME_SEC_FIELD, NumericType.INT32);
+ timeSecField.SetInt32Value(sec);
+ doc.Add(timeSecField);
+
+ // Set TITLE_FIELD
+ string title = docData.Title;
+ Field titleField = ds.GetField(TITLE_FIELD, m_valType);
+ titleField.SetStringValue(title == null ? "" : title);
+ doc.Add(titleField);
+
+ string body = docData.Body;
+ if (body != null && body.Length > 0)
+ {
+ string bdy;
+ if (size <= 0 || size >= body.Length)
+ {
+ bdy = body; // use all
+ docData.Body = ""; // nothing left
+ }
+ else
+ {
+ // attempt not to break words - if whitespace found within next 20 chars...
+ for (int n = size - 1; n < size + 20 && n < body.Length; n++)
+ {
+ if (char.IsWhiteSpace(body[n]))
+ {
+ size = n;
+ break;
+ }
+ }
+ bdy = body.Substring(0, size - 0); // use part
+ docData.Body = body.Substring(size); // some left
+ }
+ Field bodyField = ds.GetField(BODY_FIELD, m_bodyValType);
+ bodyField.SetStringValue(bdy);
+ doc.Add(bodyField);
+
+ if (storeBytes)
+ {
+ Field bytesField = ds.GetField(BYTES_FIELD, StringField.TYPE_STORED);
+ bytesField.SetBytesValue(Encoding.UTF8.GetBytes(bdy));
+ doc.Add(bytesField);
+ }
+ }
+
+ if (m_indexProperties)
+ {
+ var props = docData.Props;
+ if (props != null)
+ {
+ foreach (var entry in props)
+ {
+ Field f = ds.GetField((string)entry.Key, m_valType);
+ f.SetStringValue((string)entry.Value);
+ doc.Add(f);
+ }
+ docData.Props = null;
+ }
+ }
+
+ //System.out.println("============== Created doc "+numDocsCreated+" :\n"+doc+"\n==========");
+ return doc;
+ }
+
+ private void ResetLeftovers()
+ {
+ leftovr.Value = null;
+ }
+
+ protected virtual DocState GetDocState()
+ {
+ DocState ds = docState.Value;
+ if (ds == null)
+ {
+ ds = new DocState(m_reuseFields, m_valType, m_bodyValType);
+ docState.Value = ds;
+ }
+ return ds;
+ }
+
+ /// <summary>
+ /// Closes the <see cref="DocMaker"/>.
+ /// </summary>
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ /// <summary>
+ /// Closes the <see cref="DocMaker"/>. The base implementation closes the
+ /// <see cref="ContentSource"/>, and it can be overridden to do more work (but make
+ /// sure to call <c>base.Dispose(bool)</c>).
+ /// </summary>
+ protected virtual void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ m_source.Dispose();
+ }
+ }
+
+ /// <summary>
+ /// Creates a <see cref="Document"/> object ready for indexing. This method uses the
+ /// <see cref="ContentSource"/> to get the next document from the source, and creates
+ /// a <see cref="Document"/> object from the returned fields. If
+ /// <c>reuseFields</c> was set to <c>true</c>, it will reuse <see cref="Document"/>
+ /// and <see cref="Field"/> instances.
+ /// </summary>
+ /// <returns></returns>
+ public virtual Document MakeDocument()
+ {
+ ResetLeftovers();
+ DocData docData = m_source.GetNextDocData(GetDocState().docData);
+ Document doc = CreateDocument(docData, 0, -1);
+ return doc;
+ }
+
+ /// <summary>
+ /// Same as <see cref="MakeDocument()"/>, only this method creates a document of the
+ /// given size input by <paramref name="size"/>.
+ /// </summary>
+ public virtual Document MakeDocument(int size)
+ {
+ LeftOver lvr = leftovr.Value;
+ if (lvr == null || lvr.DocData == null || lvr.DocData.Body == null
+ || lvr.DocData.Body.Length == 0)
+ {
+ ResetLeftovers();
+ }
+ DocData docData = GetDocState().docData;
+ DocData dd = (lvr == null ? m_source.GetNextDocData(docData) : lvr.DocData);
+ int cnt = (lvr == null ? 0 : lvr.Count);
+ while (dd.Body == null || dd.Body.Length < size)
+ {
+ DocData dd2 = dd;
+ dd = m_source.GetNextDocData(new DocData());
+ cnt = 0;
+ dd.Body = (dd2.Body + dd.Body);
+ }
+ Document doc = CreateDocument(dd, size, cnt);
+ if (dd.Body == null || dd.Body.Length == 0)
+ {
+ ResetLeftovers();
+ }
+ else
+ {
+ if (lvr == null)
+ {
+ lvr = new LeftOver();
+ leftovr.Value = lvr;
+ }
+ lvr.DocData = dd;
+ lvr.Count = ++cnt;
+ }
+ return doc;
+ }
+
+ /// <summary>Reset inputs so that the test run would behave, input wise, as if it just started.</summary>
+ public virtual void ResetInputs()
+ {
+ m_source.PrintStatistics("docs");
+ // re-initiate since properties by round may have changed.
+ SetConfig(m_config, m_source);
+ m_source.ResetInputs();
+ numDocsCreated.Set(0);
+ ResetLeftovers();
+ }
+
+ /// <summary>Set the configuration parameters of this doc maker.</summary>
+ public virtual void SetConfig(Config config, ContentSource source)
+ {
+ this.m_config = config;
+ this.m_source = source;
+
+ bool stored = config.Get("doc.stored", false);
+ bool bodyStored = config.Get("doc.body.stored", stored);
+ bool tokenized = config.Get("doc.tokenized", true);
+ bool bodyTokenized = config.Get("doc.body.tokenized", tokenized);
+ bool norms = config.Get("doc.tokenized.norms", false);
+ bool bodyNorms = config.Get("doc.body.tokenized.norms", true);
+ bool termVec = config.Get("doc.term.vector", false);
+ bool termVecPositions = config.Get("doc.term.vector.positions", false);
+ bool termVecOffsets = config.Get("doc.term.vector.offsets", false);
+
+ m_valType = new FieldType(TextField.TYPE_NOT_STORED);
+ m_valType.IsStored = stored;
+ m_valType.IsTokenized = tokenized;
+ m_valType.OmitNorms = !norms;
+ m_valType.StoreTermVectors = termVec;
+ m_valType.StoreTermVectorPositions = termVecPositions;
+ m_valType.StoreTermVectorOffsets = termVecOffsets;
+ m_valType.Freeze();
+
+ m_bodyValType = new FieldType(TextField.TYPE_NOT_STORED);
+ m_bodyValType.IsStored = bodyStored;
+ m_bodyValType.IsTokenized = bodyTokenized;
+ m_bodyValType.OmitNorms = !bodyNorms;
+ m_bodyValType.StoreTermVectors = termVec;
+ m_bodyValType.StoreTermVectorPositions = termVecPositions;
+ m_bodyValType.StoreTermVectorOffsets = termVecOffsets;
+ m_bodyValType.Freeze();
+
+ storeBytes = config.Get("doc.store.body.bytes", false);
+
+ m_reuseFields = config.Get("doc.reuse.fields", true);
+
+ // In a multi-rounds run, it is important to reset DocState since settings
+ // of fields may change between rounds, and this is the only way to reset
+ // the cache of all threads.
+ docState = new ThreadLocal<DocState>();
+
+ m_indexProperties = config.Get("doc.index.props", false);
+
+ updateDocIDLimit = config.Get("doc.random.id.limit", -1);
+ if (updateDocIDLimit != -1)
+ {
+ r = new Random(179);
+ }
+ }
+ }
+}
[29/33] lucenenet git commit: Merge branch 'master' into benchmark
Posted by ni...@apache.org.
Merge branch 'master' into benchmark
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/0bc62b57
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/0bc62b57
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/0bc62b57
Branch: refs/heads/master
Commit: 0bc62b5702c3168804e6abb569d845349d27c6ed
Parents: acc82da ea879c6
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Aug 4 20:56:51 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 20:56:51 2017 +0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[13/33] lucenenet git commit:
Lucene.Net.Benchmark.ByTask.Feeds.EnwikiContentSource: Fixed concurrency bugs
that were causing tests not to finish
Posted by ni...@apache.org.
Lucene.Net.Benchmark.ByTask.Feeds.EnwikiContentSource: Fixed concurrency bugs that were causing tests not to finish
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/a60c5ef4
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/a60c5ef4
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/a60c5ef4
Branch: refs/heads/master
Commit: a60c5ef4d1c2e4f617fc732cdcf42f3df2a15a83
Parents: b515271
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Tue Aug 1 08:08:19 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Wed Aug 2 09:55:04 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs | 6 +++---
.../ByTask/Feeds/EnwikiContentSourceTest.cs | 1 -
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a60c5ef4/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
index c9db5a2..2cb24d0 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
@@ -68,7 +68,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
if (t == null)
{
threadDone = false;
- t = new ThreadClass(/*this*/);
+ t = new ThreadClass(Run);
t.SetDaemon(true);
t.Start();
}
@@ -165,7 +165,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
case BODY:
body = contents.ToString();
//workaround that startswith doesn't have an ignore case option, get at least 20 chars.
- string startsWith = body.Substring(0, Math.Min(10, contents.Length)).ToLowerInvariant();
+ string startsWith = body.Substring(0, Math.Min(10, contents.Length) - 0).ToLowerInvariant();
if (startsWith.StartsWith("#redirect", StringComparison.Ordinal))
{
body = null;
@@ -195,7 +195,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
try
{
- Sax.Net.IXmlReader reader = XmlReaderFactory.Current.CreateXmlReader(); //XMLReaderFactory.createXMLReader();
+ Sax.Net.IXmlReader reader = new TagSoup.Net.XmlReaderFactory().CreateXmlReader(); //XMLReaderFactory.createXMLReader();
reader.ContentHandler = this;
reader.ErrorHandler = this;
while (!stopped)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a60c5ef4/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
index 95ded38..220d75e 100644
--- a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
@@ -26,7 +26,6 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
* limitations under the License.
*/
- [Ignore("LUCENENET TODO: Never finishes")]
public class EnwikiContentSourceTest : LuceneTestCase
{
/** An EnwikiContentSource which works on a String and not files. */
[16/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.rnc
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.rnc b/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.rnc
new file mode 100644
index 0000000..b767640
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.rnc
@@ -0,0 +1,49 @@
+# This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+#
+# TagSoup is licensed under the Apache License,
+# Version 2.0. You may obtain a copy of this license at
+# http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+# additional legal rights not granted by this license.
+#
+# TagSoup is distributed in the hope that it will be useful, but
+# unless required by applicable law or agreed to in writing, TagSoup
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+# OF ANY KIND, either express or implied; not even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# State Table Markup Language
+default namespace = "http://www.ccil.org/~cowan/XML/tagsoup/stml"
+start = statetable
+
+## Root element
+statetable = element statetable {
+ attribute version {"1.0"},
+ attribute id {xsd:ID}?,
+ symbol*,
+ action+,
+ state+
+ }
+
+## A symbol specifies a non-character input to the state machine
+symbol = element symbol {
+ attribute id {xsd:ID}
+ }
+
+## An action is the name for what the state machine does
+action = element action {
+ attribute id {xsd:ID}
+ }
+
+## A state specifies an internal state of the machine
+state = element state {
+ attribute id {xsd:ID},
+ tr*
+ }
+
+## A tr specifies a state transition
+tr = element tr {
+ (attribute symbol {xsd:IDREF} |
+ attribute char {xsd:string {length = "1"}}),
+ attribute action {xsd:IDREF},
+ attribute newstate {xsd:IDREF}
+ }
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.xslt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.xslt b/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.xslt
new file mode 100644
index 0000000..524f998
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/stml/stml.xslt
@@ -0,0 +1,150 @@
+<!--
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+-->
+
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:stml="http://www.ccil.org/~cowan/XML/tagsoup/stml"
+ version="1.0">
+
+ <xsl:output method="text"/>
+
+ <xsl:strip-space elements="*"/>
+
+ <!-- The main template. Generates declarations for states and
+ actions, then the statetable itself, and then a comment (used for
+ manual checking) listing all the actions compactly. -->
+ <xsl:template match="stml:statetable">
+ <xsl:apply-templates select="stml:state">
+ <xsl:sort select="@id"/>
+ </xsl:apply-templates>
+
+ <xsl:apply-templates select="stml:action">
+ <xsl:sort select="@id"/>
+ </xsl:apply-templates>
+
+ <xsl:text>	private static int[] statetable = {
</xsl:text>
+ <xsl:apply-templates select="stml:state/stml:tr">
+ <xsl:sort select="../@id"/>
+ <xsl:sort select="@symbol"/>
+ <xsl:sort select="@char"/>
+ </xsl:apply-templates>
+ <xsl:text>
	};
</xsl:text>
+
+ <xsl:text>	private static readonly string[] debug_actionnames = { ""</xsl:text>
+ <xsl:apply-templates select="stml:action" mode="debug">
+ <xsl:sort select="@id"/>
+ </xsl:apply-templates>
+ <xsl:text>};
</xsl:text>
+
+ <xsl:text>	private static readonly string[] debug_statenames = { ""</xsl:text>
+ <xsl:apply-templates select="stml:state" mode="debug">
+ <xsl:sort select="@id"/>
+ </xsl:apply-templates>
+ <xsl:text>};
</xsl:text>
+
+ <xsl:text>
</xsl:text>
+ </xsl:template>
+
+ <!-- Generate a single state declaration. -->
+ <xsl:template match="stml:state">
+ <xsl:text>	private const int </xsl:text>
+ <xsl:value-of select="@id"/>
+ <xsl:text> = </xsl:text>
+ <xsl:value-of select="position()"/>
+ <xsl:text>;
</xsl:text>
+ </xsl:template>
+
+ <!-- Generate a single action declaration. -->
+ <xsl:template match="stml:action">
+ <xsl:text>	private const int </xsl:text>
+ <xsl:value-of select="@id"/>
+ <xsl:text> = </xsl:text>
+ <xsl:value-of select="position()"/>
+ <xsl:text>;
</xsl:text>
+ </xsl:template>
+
+ <!-- Generate a single row of the statetable. -->
+ <xsl:template match="stml:tr">
+ <xsl:choose>
+ <xsl:when test="@symbol = 'EOF'">
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""-1""/>
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:when test="@symbol = 'LF'">
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""'\n'""/>
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:when test="@symbol = 'default'">
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""0""/>
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:when test="@char = "'"">
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""'\''""/>
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:when test="@symbol = 'S'">
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""' '""/>
+ </xsl:call-template>
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""'\n'""/>
+ </xsl:call-template>
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char" select=""'\t'""/>
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:call-template name="dump-tr">
+ <xsl:with-param name="char"
+ select="concat("'", @char, "'")"/>
+ </xsl:call-template>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:template>
+
+ <!-- This is a subroutine used to do the actual printing. -->
+ <xsl:template name="dump-tr">
+ <xsl:param name="char"/>
+ <xsl:text>		</xsl:text>
+ <xsl:value-of select="../@id"/>
+ <xsl:text>, </xsl:text>
+ <xsl:value-of select="$char"/>
+ <xsl:text>, </xsl:text>
+ <xsl:value-of select="@action"/>
+ <xsl:text>, </xsl:text>
+ <xsl:value-of select="@newstate"/>
+ <xsl:text>,
</xsl:text>
+ </xsl:template>
+
+ <!-- Generate a single action name in the "Actions:" comment.
+ The mode is used to keep XSLT from confusing this with the
+ regular actions template that does the action declarations. -->
+ <xsl:template match="stml:action" mode="debug">
+ <xsl:text>, "</xsl:text>
+ <xsl:value-of select="@id"/>
+ <xsl:text>"</xsl:text>
+ </xsl:template>
+
+ <!-- Generate a single stat debug name. -->
+ <xsl:template match="stml:state" mode="debug">
+ <xsl:text>, "</xsl:text>
+ <xsl:value-of select="@id"/>
+ <xsl:text>"</xsl:text>
+ </xsl:template>
+
+</xsl:transform>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-models.xslt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-models.xslt b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-models.xslt
new file mode 100644
index 0000000..7e49680
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-models.xslt
@@ -0,0 +1,47 @@
+<!-- Generate Java code to be inserted into HTMLModels.java. -->
+
+<!--
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+-->
+
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:tssl="http://www.ccil.org/~cowan/XML/tagsoup/tssl"
+ version="1.0">
+
+ <xsl:output method="text"/>
+
+ <xsl:strip-space elements="*"/>
+
+ <!-- The main template. We are going to generate Java constant
+ definitions for the groups in the file. -->
+ <xsl:template match="tssl:schema">
+ <xsl:apply-templates select="tssl:group">
+ <xsl:sort select="@id"/>
+ </xsl:apply-templates>
+ </xsl:template>
+
+ <!-- Generate a declaration for a single group. -->
+ <xsl:template match="tssl:group" name="tssl:group">
+ <xsl:param name="id" select="@id"/>
+ <xsl:param name="number" select="position()"/>
+ <xsl:text>	public const int </xsl:text>
+ <xsl:value-of select="$id"/>
+ <xsl:text> = 1 << </xsl:text>
+ <xsl:value-of select="$number"/>
+ <xsl:text>;
</xsl:text>
+ </xsl:template>
+
+</xsl:transform>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-validate.xslt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-validate.xslt b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-validate.xslt
new file mode 100644
index 0000000..81faab5
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl-validate.xslt
@@ -0,0 +1,40 @@
+<!-- Generate complaints if the schema is invalid in some way. -->
+
+<!--
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+-->
+
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:tssl="http://www.ccil.org/~cowan/XML/tagsoup/tssl"
+ version="1.0">
+
+ <xsl:output method="text"/>
+
+ <xsl:strip-space elements="*"/>
+
+ <!-- Generates a report if an element does not belong to at least
+ one of the groups that its parent element contains. -->
+ <xsl:template match="tssl:element/tssl:element">
+ <xsl:if test="not(tssl:memberOfAny) and not(tssl:memberOf/@group = ../tssl:contains/@group)">
+ <xsl:value-of select="@name"/>
+ <xsl:text> is not in the content model of </xsl:text>
+ <xsl:value-of select="../@name"/>
+ <xsl:text>
</xsl:text>
+ </xsl:if>
+ <xsl:apply-templates/>
+ </xsl:template>
+
+
+
+</xsl:transform>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.rnc
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.rnc b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.rnc
new file mode 100644
index 0000000..4443073
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.rnc
@@ -0,0 +1,75 @@
+# This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+#
+# TagSoup is licensed under the Apache License,
+# Version 2.0. You may obtain a copy of this license at
+# http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+# additional legal rights not granted by this license.
+#
+# TagSoup is distributed in the hope that it will be useful, but
+# unless required by applicable law or agreed to in writing, TagSoup
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+# OF ANY KIND, either express or implied; not even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+default namespace = "http://www.ccil.org/~cowan/XML/tagsoup/tssl"
+
+start = schema
+
+## A TSSL schema has a namespace, which is applied to the elements by default,
+## and a name, which is used solely for documentation. It contains entity
+## definitions, a root element, and attributes to be applied to all elements.
+schema = element schema {
+ attribute ns {xsd:anyURI},
+ attribute prefix {xsd:NCName},
+ attribute name {xsd:NCName},
+ attribute version {"1.0"},
+ entity*,
+ group+,
+ \element,
+ \attribute*
+ }
+
+## An entity has a name and a Unicode codepoint in hex.
+entity = element entity {
+ attribute name {xsd:NCName},
+ attribute codepoint {xsd:string}
+ }
+
+## A group is a named group of elements. Every element belongs to one
+## or more groups and has a content model consisting of one or more groups.
+group = element group {
+ attribute id {xsd:ID}
+ }
+
+## An element has a name and a namespace (currently ignored).
+## It can have any of several types of content and can be restartable
+## or not. The element is also a member of one or more model groups
+## (with arbitrary names), and can contain as children zero or more
+## model groups. Elements also have attributes and "natural" children.
+\element = element element {
+ attribute ns {xsd:anyURI}?,
+ attribute name {xsd:NCName},
+ attribute type {type},
+ attribute closeMode { "unclosable" | "restartable" }?,
+ attribute text-parent { "true" | "false" }?,
+ (element memberOf { attribute group {xsd:IDREF}}+ |
+ element isRoot { empty} |
+ element memberOfAny { empty }),
+ element contains { attribute group {xsd:IDREF}}*,
+ \attribute*,
+ \element*
+ }
+
+## Here are the attribute types:
+
+type = "element" | "any" | "empty" | "mixed" | "string" | "cdata"
+
+## An attribute has a name and a namespace (currently not supported).
+## It also has an optional type and an optional default value.
+\attribute = element attribute {
+ attribute ns {xsd:anyURI}?,
+ attribute name {xsd:NCName},
+ attribute type {"ID" | "IDREF" | "IDREFS" | "NMTOKEN" | "BOOLEAN"}?,
+ attribute default {xsd:string}?
+ }
+
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.xslt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.xslt b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.xslt
new file mode 100644
index 0000000..22233d8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/tssl/tssl.xslt
@@ -0,0 +1,220 @@
+<!-- Generate Java code to be inserted into HTMLSchema.java. -->
+
+<!--
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+-->
+
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:tssl="http://www.ccil.org/~cowan/XML/tagsoup/tssl"
+ version="1.0">
+
+ <xsl:output method="text"/>
+
+ <xsl:strip-space elements="*"/>
+
+ <!-- The main template. This generates calls on the Schema routines
+ setURI(), setPrefix(), elementType(), parent(), attribute(),
+ and entity() in that order. Several special cases are
+ handled by template calls. -->
+ <xsl:template match="tssl:schema">
+ <!-- setURI() -->
+ <xsl:text>		Uri = "</xsl:text>
+ <xsl:value-of select="@ns"/>
+ <xsl:text>";
</xsl:text>
+ <!-- setPrefix() -->
+ <xsl:text>		Prefix = "</xsl:text>
+ <xsl:value-of select="@prefix"/>
+ <xsl:text>";
</xsl:text>
+ <!-- elementType() special cases -->
+ <xsl:text>		ElementType("<pcdata>", M_EMPTY, M_PCDATA, 0);
</xsl:text>
+ <xsl:text>		ElementType("<root>", </xsl:text>
+ <xsl:apply-templates select="tssl:element/tssl:isRoot"/>
+ <xsl:text>, M_EMPTY, 0);
</xsl:text>
+ <!-- elementType() main loop -->
+ <xsl:apply-templates select="//tssl:element">
+ <xsl:sort select="@name"/>
+ </xsl:apply-templates>
+ <!-- parent() special cases -->
+ <xsl:call-template name="parent">
+ <xsl:with-param name="elem" select="'<pcdata>'"/>
+ <xsl:with-param name="parent" select="//tssl:element[@text-parent='true']/@name"/>
+ </xsl:call-template>
+ <xsl:call-template name="parent">
+ <xsl:with-param name="elem" select="tssl:element/@name"/>
+ <xsl:with-param name="parent" select="'<root>'"/>
+ </xsl:call-template>
+ <!-- parent() main loop -->
+ <xsl:apply-templates select="//tssl:element/tssl:element" mode="parent">
+ <xsl:sort select="@name"/>
+ </xsl:apply-templates>
+ <xsl:apply-templates select="//tssl:element/tssl:attribute">
+ <xsl:sort select="../@name"/>
+ <xsl:sort select="@name"/>
+ </xsl:apply-templates>
+ <!-- attribute() main loop -->
+ <xsl:apply-templates select="tssl:attribute">
+ <xsl:sort select="@name"/>
+ </xsl:apply-templates>
+ <!-- entity() main loop -->
+ <xsl:apply-templates select="tssl:entity">
+ <xsl:sort select="@name"/>
+ </xsl:apply-templates>
+ </xsl:template>
+
+ <!-- Generates a single call to elementType(). -->
+ <xsl:template match="tssl:element">
+ <xsl:text>		ElementType("</xsl:text>
+ <xsl:value-of select="@name"/>
+ <xsl:text>", </xsl:text>
+ <xsl:choose>
+ <xsl:when test="@type = 'element'">
+ <xsl:apply-templates select="tssl:contains"/>
+ </xsl:when>
+ <xsl:when test="@type = 'string'">
+ <xsl:text>M_PCDATA</xsl:text>
+ </xsl:when>
+ <xsl:when test="@type = 'mixed'">
+ <xsl:text>M_PCDATA|</xsl:text>
+ <xsl:apply-templates select="tssl:contains"/>
+ </xsl:when>
+ <xsl:when test="@type = 'empty'">
+ <xsl:text>M_EMPTY</xsl:text>
+ </xsl:when>
+ <xsl:when test="@type = 'any'">
+ <xsl:text>M_ANY</xsl:text>
+ </xsl:when>
+ <xsl:when test="@type = 'cdata'">
+ <xsl:text>M_PCDATA</xsl:text>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:text>, </xsl:text>
+ <xsl:apply-templates select="tssl:memberOf"/>
+ <xsl:apply-templates select="tssl:memberOfAny"/>
+ <xsl:apply-templates select="tssl:isRoot"/>
+ <xsl:text>, </xsl:text>
+ <xsl:choose>
+ <xsl:when test="@type = 'cdata'">
+ <xsl:text>F_CDATA</xsl:text>
+ </xsl:when>
+ <xsl:when test="@closeMode = 'restartable'">
+ <xsl:text>F_RESTART</xsl:text>
+ </xsl:when>
+ <xsl:when test="@closeMode = 'unclosable'">
+ <xsl:text>F_NOFORCE</xsl:text>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:text>0</xsl:text>
+ </xsl:otherwise>
+ </xsl:choose>
+ <xsl:text>);
</xsl:text>
+ </xsl:template>
+
+ <!-- Applied from tssl:element to generate the contains argument. -->
+ <xsl:template match="tssl:contains">
+ <xsl:value-of select="@group"/>
+ <xsl:if test="position() != last()">
+ <xsl:text>|</xsl:text>
+ </xsl:if>
+ </xsl:template>
+
+ <!-- Applied from tssl:element to generate the memberOf argument. -->
+ <xsl:template match="tssl:memberOf">
+ <xsl:value-of select="@group"/>
+ <xsl:if test="position() != last()">
+ <xsl:text>|</xsl:text>
+ </xsl:if>
+ </xsl:template>
+
+ <!-- Applied from tssl:element to handle memberOfAny. -->
+ <xsl:template match="tssl:memberOfAny">
+ <xsl:text>(int)(M_ANY & ~M_ROOT)</xsl:text>
+ </xsl:template>
+
+ <!-- Applied from tssl:element to handle isRoot. -->
+ <xsl:template match="tssl:isRoot">
+ <xsl:text>M_ROOT</xsl:text>
+ </xsl:template>
+
+ <!-- Generates a single call to parent(). The mode is used to prevent XSLT
+ from getting confused and generating elementType calls instead. -->
+ <xsl:template match="tssl:element/tssl:element" name="parent" mode="parent">
+ <xsl:param name="elem" select="@name"/>
+ <xsl:param name="parent" select="../@name"/>
+ <xsl:text>		Parent("</xsl:text>
+ <xsl:value-of select="$elem"/>
+ <xsl:text>", "</xsl:text>
+ <xsl:value-of select="$parent"/>
+ <xsl:text>");
</xsl:text>
+ </xsl:template>
+
+ <!-- Generates a single call to attribute(). -->
+ <xsl:template match="tssl:element/tssl:attribute" name="tssl:attribute">
+ <xsl:param name="elem" select="../@name"/>
+ <xsl:param name="attr" select="@name"/>
+ <xsl:param name="type" select="@type"/>
+ <xsl:param name="default" select="@default"/>
+ <xsl:text>		Attribute("</xsl:text>
+ <xsl:value-of select="$elem"/>
+ <xsl:text>", "</xsl:text>
+ <xsl:value-of select="$attr"/>
+ <xsl:text>", "</xsl:text>
+ <xsl:choose>
+ <xsl:when test="$type">
+ <xsl:value-of select="$type"/>
+ </xsl:when>
+ <xsl:when test="not($type)">
+ <xsl:text>CDATA</xsl:text>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:text>", </xsl:text>
+ <xsl:choose>
+ <xsl:when test="$default">
+ <xsl:text>"</xsl:text>
+ <xsl:value-of select="$default"/>
+ <xsl:text>"</xsl:text>
+ </xsl:when>
+ <xsl:when test="not($default)">
+ <xsl:text>null</xsl:text>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:text>);
</xsl:text>
+ </xsl:template>
+
+ <!-- Generates calls to attribute() (using the above template)
+ based on the global attribute definitions. -->
+ <xsl:template match="tssl:schema/tssl:attribute">
+ <xsl:variable name="attr" select="@name"/>
+ <xsl:variable name="type" select="@type"/>
+ <xsl:variable name="default" select="@default"/>
+ <xsl:for-each select="//tssl:element">
+ <xsl:sort select="@name"/>
+ <xsl:call-template name="tssl:attribute">
+ <xsl:with-param name="elem" select="@name"/>
+ <xsl:with-param name="attr" select="$attr"/>
+ <xsl:with-param name="type" select="$type"/>
+ <xsl:with-param name="default" select="$default"/>
+ </xsl:call-template>
+ </xsl:for-each>
+ </xsl:template>
+
+ <!-- Generates a single call to entity(). -->
+ <xsl:template match="tssl:entity">
+ <xsl:text>		Entity("</xsl:text>
+ <xsl:value-of select="@name"/>
+ <xsl:text>", 0x</xsl:text>
+ <xsl:value-of select="@codepoint"/>
+ <xsl:text>);
</xsl:text>
+ </xsl:template>
+
+</xsl:transform>
[04/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Trec/Trec1MQReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Trec/Trec1MQReader.cs b/src/Lucene.Net.Benchmark/Quality/Trec/Trec1MQReader.cs
new file mode 100644
index 0000000..85dceda
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Trec/Trec1MQReader.cs
@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Lucene.Net.Benchmarks.Quality.Trec
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Read topics of TREC 1MQ track.
+ /// <para/>
+ /// Expects this topic format -
+ /// <code>
+ /// qnum:qtext
+ /// </code>
+ /// Comment lines starting with '#' are ignored.
+ /// <para/>
+ /// All topics will have a single name value pair.
+ /// </summary>
+ public class Trec1MQReader
+ {
+ private string name;
+
+ /// <summary>
+ /// Constructor for Trec's 1MQ TopicsReader
+ /// </summary>
+ /// <param name="name">Name of name-value pair to set for all queries.</param>
+ public Trec1MQReader(string name)
+ : base()
+ {
+ this.name = name;
+ }
+
+ /// <summary>
+ /// Read quality queries from trec 1MQ format topics file.
+ /// </summary>
+ /// <param name="reader">where queries are read from.</param>
+ /// <returns>the result quality queries.</returns>
+ /// <exception cref="IOException">if cannot read the queries.</exception>
+ public virtual QualityQuery[] ReadQueries(TextReader reader)
+ {
+ IList<QualityQuery> res = new List<QualityQuery>();
+ string line;
+ try
+ {
+ while (null != (line = reader.ReadLine()))
+ {
+ line = line.Trim();
+ if (line.StartsWith("#", StringComparison.Ordinal))
+ {
+ continue;
+ }
+ // id
+ int k = line.IndexOf(':');
+ string id = line.Substring(0, k - 0).Trim();
+ // qtext
+ string qtext = line.Substring(k + 1).Trim();
+ // we got a topic!
+ IDictionary<string, string> fields = new Dictionary<string, string>();
+ fields[name] = qtext;
+ //System.out.println("id: "+id+" qtext: "+qtext+" line: "+line);
+ QualityQuery topic = new QualityQuery(id, fields);
+ res.Add(topic);
+ }
+ }
+ finally
+ {
+ reader.Dispose();
+ }
+ // sort result array (by ID)
+ QualityQuery[] qq = res.ToArray();
+ Array.Sort(qq);
+ return qq;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Trec/TrecJudge.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Trec/TrecJudge.cs b/src/Lucene.Net.Benchmark/Quality/Trec/TrecJudge.cs
new file mode 100644
index 0000000..386b130
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Trec/TrecJudge.cs
@@ -0,0 +1,186 @@
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.Quality.Trec
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Judge if given document is relevant to given quality query, based on Trec format for judgements.
+ /// </summary>
+ public class TrecJudge : IJudge
+ {
+ IDictionary<string, QRelJudgement> judgements;
+
+ /// <summary>
+ /// Constructor from a reader.
+ /// </summary>
+ /// <remarks>
+ /// Expected input format:
+ /// <code>
+ /// qnum 0 doc-name is-relevant
+ /// </code>
+ /// Two sample lines:
+ /// <code>
+ /// 19 0 doc303 1
+ /// 19 0 doc7295 0
+ /// </code>
+ /// </remarks>
+ /// <param name="reader">Where judgments are read from.</param>
+ /// <exception cref="IOException">If there is a low-level I/O error.</exception>
+ public TrecJudge(TextReader reader)
+ {
+ judgements = new Dictionary<string, QRelJudgement>();
+ QRelJudgement curr = null;
+ string zero = "0";
+ string line;
+
+ try
+ {
+ while (null != (line = reader.ReadLine()))
+ {
+ line = line.Trim();
+ if (line.Length == 0 || '#' == line[0])
+ {
+ continue;
+ }
+ StringTokenizer st = new StringTokenizer(line);
+ string queryID = st.NextToken();
+ st.NextToken();
+ string docName = st.NextToken();
+ bool relevant = !zero.Equals(st.NextToken(), StringComparison.Ordinal);
+ // LUCENENET: don't call st.NextToken() unless the condition fails.
+ Debug.Assert(!st.HasMoreTokens(), "wrong format: " + line + " next: " + (st.HasMoreTokens() ? st.NextToken() : ""));
+ if (relevant)
+ { // only keep relevant docs
+ if (curr == null || !curr.queryID.Equals(queryID, StringComparison.Ordinal))
+ {
+ if (!judgements.TryGetValue(queryID, out curr) || curr == null)
+ {
+ curr = new QRelJudgement(queryID);
+ judgements[queryID] = curr;
+ }
+ }
+ curr.AddRelevantDoc(docName);
+ }
+ }
+ }
+ finally
+ {
+ reader.Dispose();
+ }
+ }
+
+ // inherit javadocs
+ public virtual bool IsRelevant(string docName, QualityQuery query)
+ {
+ QRelJudgement qrj;// = judgements.get(query.getQueryID());
+ judgements.TryGetValue(query.QueryID, out qrj);
+ return qrj != null && qrj.IsRelevant(docName);
+ }
+
+ /// <summary>
+ /// Single Judgement of a trec quality query.
+ /// </summary>
+ private class QRelJudgement
+ {
+ internal string queryID;
+ private IDictionary<string, string> relevantDocs;
+
+ internal QRelJudgement(string queryID)
+ {
+ this.queryID = queryID;
+ relevantDocs = new HashMap<string, string>();
+ }
+
+ public virtual void AddRelevantDoc(string docName)
+ {
+ relevantDocs[docName] = docName;
+ }
+
+ internal virtual bool IsRelevant(string docName)
+ {
+ return relevantDocs.ContainsKey(docName);
+ }
+
+ public virtual int MaxRecall
+ {
+ get { return relevantDocs.Count; }
+ }
+ }
+
+ // inherit javadocs
+ public virtual bool ValidateData(QualityQuery[] qq, TextWriter logger)
+ {
+ IDictionary<string, QRelJudgement> missingQueries = new Dictionary<string, QRelJudgement>(judgements);
+ IList<string> missingJudgements = new List<string>();
+ for (int i = 0; i < qq.Length; i++)
+ {
+ string id = qq[i].QueryID;
+ if (missingQueries.ContainsKey(id))
+ {
+ missingQueries.Remove(id);
+ }
+ else
+ {
+ missingJudgements.Add(id);
+ }
+ }
+ bool isValid = true;
+ if (missingJudgements.Count > 0)
+ {
+ isValid = false;
+ if (logger != null)
+ {
+ logger.WriteLine("WARNING: " + missingJudgements.Count + " queries have no judgments! - ");
+ for (int i = 0; i < missingJudgements.Count; i++)
+ {
+ logger.WriteLine(" " + missingJudgements[i]);
+ }
+ }
+ }
+ if (missingQueries.Count > 0)
+ {
+ isValid = false;
+ if (logger != null)
+ {
+ logger.WriteLine("WARNING: " + missingQueries.Count + " judgments match no query! - ");
+ foreach (string id in missingQueries.Keys)
+ {
+ logger.WriteLine(" " + id);
+ }
+ }
+ }
+ return isValid;
+ }
+
+ // inherit javadocs
+ public virtual int MaxRecall(QualityQuery query)
+ {
+ QRelJudgement qrj;
+ if (judgements.TryGetValue(query.QueryID, out qrj) && qrj != null)
+ {
+ return qrj.MaxRecall;
+ }
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Trec/TrecTopicsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Trec/TrecTopicsReader.cs b/src/Lucene.Net.Benchmark/Quality/Trec/TrecTopicsReader.cs
new file mode 100644
index 0000000..158386f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Trec/TrecTopicsReader.cs
@@ -0,0 +1,154 @@
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.Quality.Trec
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Read TREC topics.
+ /// </summary>
+ /// <remarks>
+ /// Expects this topic format -
+ /// <code>
+ /// <top>
+ /// <num> Number: nnn
+ ///
+ /// <title> title of the topic
+ ///
+ /// <desc> Description:
+ /// description of the topic
+ ///
+ /// <narr> Narrative:
+ /// "story" composed by assessors.
+ ///
+ /// </top>
+ /// </code>
+ /// Comment lines starting with '#' are ignored.
+ /// </remarks>
+ public class TrecTopicsReader
+ {
+ private static readonly string newline = Environment.NewLine;
+
+ /// <summary>
+ /// Constructor for Trec's TopicsReader
+ /// </summary>
+ public TrecTopicsReader()
+ : base()
+ {
+ }
+
+ /// <summary>
+ /// Read quality queries from trec format topics file.
+ /// </summary>
+ /// <param name="reader">where queries are read from.</param>
+ /// <returns>the result quality queries.</returns>
+ /// <exception cref="IOException">if cannot read the queries.</exception>
+ public virtual QualityQuery[] ReadQueries(TextReader reader)
+ {
+ IList<QualityQuery> res = new List<QualityQuery>();
+ StringBuilder sb;
+ try
+ {
+ while (null != (sb = Read(reader, "<top>", null, false, false)))
+ {
+ IDictionary<string, string> fields = new Dictionary<string, string>();
+ // id
+ sb = Read(reader, "<num>", null, true, false);
+ int k = sb.IndexOf(":");
+ string id = sb.ToString(k + 1, sb.Length - (k + 1)).Trim();
+ // title
+ sb = Read(reader, "<title>", null, true, false);
+ k = sb.IndexOf(">");
+ string title = sb.ToString(k + 1, sb.Length - (k + 1)).Trim();
+ // description
+ Read(reader, "<desc>", null, false, false);
+ sb.Length = 0;
+ string line = null;
+ while ((line = reader.ReadLine()) != null)
+ {
+ if (line.StartsWith("<narr>", StringComparison.Ordinal))
+ break;
+ if (sb.Length > 0) sb.Append(' ');
+ sb.Append(line);
+ }
+ string description = sb.ToString().Trim();
+ // narrative
+ sb.Length = 0;
+ while ((line = reader.ReadLine()) != null)
+ {
+ if (line.StartsWith("</top>", StringComparison.Ordinal))
+ break;
+ if (sb.Length > 0) sb.Append(' ');
+ sb.Append(line);
+ }
+ string narrative = sb.ToString().Trim();
+ // we got a topic!
+ fields["title"] = title;
+ fields["description"] = description;
+ fields["narrative"] = narrative;
+ QualityQuery topic = new QualityQuery(id, fields);
+ res.Add(topic);
+ }
+ }
+ finally
+ {
+ reader.Dispose();
+ }
+ // sort result array (by ID)
+ QualityQuery[] qq = res.ToArray();
+ Array.Sort(qq);
+ return qq;
+ }
+
+ // read until finding a line that starts with the specified prefix
+ private StringBuilder Read(TextReader reader, string prefix, StringBuilder sb, bool collectMatchLine, bool collectAll)
+ {
+ sb = (sb == null ? new StringBuilder() : sb);
+ string sep = "";
+ while (true)
+ {
+ string line = reader.ReadLine();
+ if (line == null)
+ {
+ return null;
+ }
+ if (line.StartsWith(prefix, StringComparison.Ordinal))
+ {
+ if (collectMatchLine)
+ {
+ sb.Append(sep + line);
+ sep = newline;
+ }
+ break;
+ }
+ if (collectAll)
+ {
+ sb.Append(sep + line);
+ sep = newline;
+ }
+ }
+ //System.out.println("read: "+sb);
+ return sb;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Utils/DocNameExtractor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Utils/DocNameExtractor.cs b/src/Lucene.Net.Benchmark/Quality/Utils/DocNameExtractor.cs
new file mode 100644
index 0000000..6e5cc0f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Utils/DocNameExtractor.cs
@@ -0,0 +1,89 @@
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Lucene.Net.Benchmarks.Quality.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Utility: extract doc names from an index
+ /// </summary>
+ public class DocNameExtractor
+ {
+ private readonly string docNameField;
+
+ /// <summary>
+ /// Constructor for <see cref="DocNameExtractor"/>.
+ /// </summary>
+ /// <param name="docNameField">name of the stored field containing the doc name.</param>
+ public DocNameExtractor(string docNameField)
+ {
+ this.docNameField = docNameField;
+ }
+
+ /// <summary>
+ /// Extract the name of the input doc from the index.
+ /// </summary>
+ /// <param name="searcher">access to the index.</param>
+ /// <param name="docid">ID of doc whose name is needed.</param>
+ /// <returns>the name of the input doc as extracted from the index.</returns>
+ /// <exception cref="System.IO.IOException">if cannot extract the doc name from the index.</exception>
+ public virtual string DocName(IndexSearcher searcher, int docid)
+ {
+ IList<string> name = new List<string>();
+ searcher.IndexReader.Document(docid, new StoredFieldVisitorAnonymousHelper(this, name));
+
+ return name.FirstOrDefault();
+ }
+
+ private class StoredFieldVisitorAnonymousHelper : StoredFieldVisitor
+ {
+ private readonly DocNameExtractor outerInstance;
+ private readonly IList<string> name;
+
+ public StoredFieldVisitorAnonymousHelper(DocNameExtractor outerInstance, IList<string> name)
+ {
+ this.outerInstance = outerInstance;
+ this.name = name;
+ }
+ public override void StringField(FieldInfo fieldInfo, string value)
+ {
+ name.Add(value);
+ }
+
+ public override Status NeedsField(FieldInfo fieldInfo)
+ {
+ if (name.Count > 0)
+ {
+ return Status.STOP;
+ }
+ else if (fieldInfo.Name.Equals(outerInstance.docNameField, StringComparison.Ordinal))
+ {
+ return Status.YES;
+ }
+ else
+ {
+ return Status.NO;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Utils/QualityQueriesFinder.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Utils/QualityQueriesFinder.cs b/src/Lucene.Net.Benchmark/Quality/Utils/QualityQueriesFinder.cs
new file mode 100644
index 0000000..062263a
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Utils/QualityQueriesFinder.cs
@@ -0,0 +1,152 @@
+using Lucene.Net.Index;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using System;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.Quality.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Suggest Quality queries based on an index contents.
+ /// Utility class, used for making quality test benchmarks.
+ /// </summary>
+ public class QualityQueriesFinder
+ {
+ private static readonly string newline = Environment.NewLine;
+ private Store.Directory dir;
+
+ /// <summary>
+ /// Constructor over a directory containing the index.
+ /// </summary>
+ /// <param name="dir">Directory containing the index we search for the quality test.</param>
+ private QualityQueriesFinder(Store.Directory dir)
+ {
+ this.dir = dir;
+ }
+
+ /// <summary>
+ ///
+ /// </summary>
+ /// <param name="args">{index-dir}</param>
+ /// <exception cref="IOException">if cannot access the index.</exception>
+ public static void Main(string[] args)
+ {
+ if (args.Length < 1)
+ {
+ SystemConsole.Error.WriteLine("Usage: java QualityQueriesFinder <index-dir>");
+ Environment.Exit(1);
+ }
+ QualityQueriesFinder qqf = new QualityQueriesFinder(FSDirectory.Open(new DirectoryInfo(args[0])));
+ string[] q = qqf.BestQueries("body", 20);
+ for (int i = 0; i < q.Length; i++)
+ {
+ SystemConsole.WriteLine(newline + FormatQueryAsTrecTopic(i, q[i], null, null));
+ }
+ }
+
+ private string[] BestQueries(string field, int numQueries)
+ {
+ string[] words = BestTerms("body", 4 * numQueries);
+ int n = words.Length;
+ int m = n / 4;
+ string[] res = new string[m];
+ for (int i = 0; i < res.Length; i++)
+ {
+ res[i] = words[i] + " " + words[m + i] + " " + words[n - 1 - m - i] + " " + words[n - 1 - i];
+ //System.out.println("query["+i+"]: "+res[i]);
+ }
+ return res;
+ }
+
+ private static string FormatQueryAsTrecTopic(int qnum, string title, string description, string narrative)
+ {
+ return
+ "<top>" + newline +
+ "<num> Number: " + qnum + newline + newline +
+ "<title> " + (title == null ? "" : title) + newline + newline +
+ "<desc> Description:" + newline +
+ (description == null ? "" : description) + newline + newline +
+ "<narr> Narrative:" + newline +
+ (narrative == null ? "" : narrative) + newline + newline +
+ "</top>";
+ }
+
+ private string[] BestTerms(string field, int numTerms)
+ {
+ Util.PriorityQueue<TermDf> pq = new TermsDfQueue(numTerms);
+ IndexReader ir = DirectoryReader.Open(dir);
+ try
+ {
+ int threshold = ir.MaxDoc / 10; // ignore words too common.
+ Terms terms = MultiFields.GetTerms(ir, field);
+ if (terms != null)
+ {
+ TermsEnum termsEnum = terms.GetIterator(null);
+ while (termsEnum.Next() != null)
+ {
+ int df = termsEnum.DocFreq;
+ if (df < threshold)
+ {
+ string ttxt = termsEnum.Term.Utf8ToString();
+ pq.InsertWithOverflow(new TermDf(ttxt, df));
+ }
+ }
+ }
+ }
+ finally
+ {
+ ir.Dispose();
+ }
+ string[] res = new string[pq.Count];
+ int i = 0;
+ while (pq.Count > 0)
+ {
+ TermDf tdf = pq.Pop();
+ res[i++] = tdf.word;
+ SystemConsole.WriteLine(i + ". word: " + tdf.df + " " + tdf.word);
+ }
+ return res;
+ }
+
+ private class TermDf
+ {
+ internal string word;
+ internal int df;
+ internal TermDf(string word, int freq)
+ {
+ this.word = word;
+ this.df = freq;
+ }
+ }
+
+ private class TermsDfQueue : Util.PriorityQueue<TermDf>
+ {
+ internal TermsDfQueue(int maxSize)
+ : base(maxSize)
+ {
+ }
+
+ protected override bool LessThan(TermDf tf1, TermDf tf2)
+ {
+ return tf1.df < tf2.df;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Utils/SimpleQQParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Utils/SimpleQQParser.cs b/src/Lucene.Net.Benchmark/Quality/Utils/SimpleQQParser.cs
new file mode 100644
index 0000000..0711e86
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Utils/SimpleQQParser.cs
@@ -0,0 +1,76 @@
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Util;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.Quality.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simplistic quality query parser. A Lucene query is created by passing
+ /// the value of the specified <see cref="QualityQuery"/> name-value pair(s) into
+ /// a Lucene's <see cref="QueryParser"/> using <see cref="StandardAnalyzer"/>.
+ /// </summary>
+ public class SimpleQQParser : IQualityQueryParser
+ {
+ private string[] qqNames;
+ private string indexField;
+ ThreadLocal<QueryParser> queryParser = new ThreadLocal<QueryParser>();
+
+ /// <summary>
+ /// Constructor of a simple qq parser.
+ /// </summary>
+ /// <param name="qqNames">Name-value pairs of quality query to use for creating the query.</param>
+ /// <param name="indexField">Corresponding index field.</param>
+ public SimpleQQParser(string[] qqNames, string indexField)
+ {
+ this.qqNames = qqNames;
+ this.indexField = indexField;
+ }
+
+ /// <summary>
+ /// Constructor of a simple qq parser.
+ /// </summary>
+ /// <param name="qqName">Name-value pair of quality query to use for creating the query.</param>
+ /// <param name="indexField">Corresponding index field.</param>
+ public SimpleQQParser(string qqName, string indexField)
+ : this(new string[] { qqName }, indexField)
+ {
+ }
+
+ /// <seealso cref="IQualityQueryParser.Parse(QualityQuery)"/>
+ public virtual Query Parse(QualityQuery qq)
+ {
+ QueryParser qp = queryParser.Value;
+ if (qp == null)
+ {
+#pragma warning disable 612, 618
+ qp = new QueryParser(LuceneVersion.LUCENE_CURRENT, indexField, new StandardAnalyzer(LuceneVersion.LUCENE_CURRENT));
+#pragma warning restore 612, 618
+ queryParser.Value = qp;
+ }
+ BooleanQuery bq = new BooleanQuery();
+ for (int i = 0; i < qqNames.Length; i++)
+ bq.Add(qp.Parse(QueryParserBase.Escape(qq.GetValue(qqNames[i]))), Occur.SHOULD);
+
+ return bq;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Utils/SubmissionReport.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Utils/SubmissionReport.cs b/src/Lucene.Net.Benchmark/Quality/Utils/SubmissionReport.cs
new file mode 100644
index 0000000..c31eddc
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Utils/SubmissionReport.cs
@@ -0,0 +1,98 @@
+using Lucene.Net.Search;
+using System;
+using System.Globalization;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.Quality.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Create a log ready for submission.
+ /// Extend this class and override
+ /// <see cref="Report(QualityQuery, TopDocs, string, IndexSearcher)"/>
+ /// to create different reports.
+ /// </summary>
+ public class SubmissionReport
+ {
+ //private NumberFormat nf;
+ private string nf;
+ private TextWriter logger;
+ private string name;
+
+ /// <summary>
+ /// Constructor for <see cref="SubmissionReport"/>.
+ /// </summary>
+ /// <param name="logger">If <c>null</c>, no submission data is created.</param>
+ /// <param name="name">Name of this run.</param>
+ public SubmissionReport(TextWriter logger, string name)
+ {
+ this.logger = logger;
+ this.name = name;
+ nf = "{0:F4}";
+ }
+
+ /// <summary>
+ /// Report a search result for a certain quality query.
+ /// </summary>
+ /// <param name="qq">quality query for which the results are reported.</param>
+ /// <param name="td">search results for the query.</param>
+ /// <param name="docNameField">stored field used for fetching the result doc name.</param>
+ /// <param name="searcher">index access for fetching doc name.</param>
+ /// <see cref="IOException">in case of a problem.</see>
+ public virtual void Report(QualityQuery qq, TopDocs td, string docNameField, IndexSearcher searcher)
+ {
+ if (logger == null)
+ {
+ return;
+ }
+ ScoreDoc[] sd = td.ScoreDocs;
+ string sep = " \t ";
+ DocNameExtractor xt = new DocNameExtractor(docNameField);
+ for (int i = 0; i < sd.Length; i++)
+ {
+ string docName = xt.DocName(searcher, sd[i].Doc);
+ logger.WriteLine(
+ qq.QueryID + sep +
+ "Q0" + sep +
+ Format(docName, 20) + sep +
+ Format("" + i, 7) + sep +
+ //nf.format(sd[i].score) + sep +
+ string.Format(nf, sd[i].Score, CultureInfo.InvariantCulture) + sep +
+ name
+ );
+ }
+ }
+
+ public virtual void Flush()
+ {
+ if (logger != null)
+ {
+ logger.Flush();
+ }
+ }
+
+ private static string padd = " ";
+ private string Format(string s, int minLen)
+ {
+ s = (s == null ? "" : s);
+ int n = Math.Max(minLen, s.Length);
+ return (s + padd).Substring(0, n - 0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Utils/ExtractReuters.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Utils/ExtractReuters.cs b/src/Lucene.Net.Benchmark/Utils/ExtractReuters.cs
new file mode 100644
index 0000000..8727fa0
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Utils/ExtractReuters.cs
@@ -0,0 +1,167 @@
+using Lucene.Net.Support;
+using System;
+using System.IO;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace Lucene.Net.Benchmarks.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body
+ /// </summary>
+ public class ExtractReuters
+ {
+ private DirectoryInfo reutersDir;
+ private DirectoryInfo outputDir;
+ private static readonly string LINE_SEPARATOR = Environment.NewLine;
+
+ public ExtractReuters(DirectoryInfo reutersDir, DirectoryInfo outputDir)
+ {
+ this.reutersDir = reutersDir;
+ this.outputDir = outputDir;
+ SystemConsole.WriteLine("Deleting all files in " + outputDir);
+ foreach (FileInfo f in outputDir.EnumerateFiles())
+ {
+ f.Delete();
+ }
+ }
+
+ public virtual void Extract()
+ {
+ FileInfo[] sgmFiles = reutersDir.GetFiles("*.sgm");
+ if (sgmFiles != null && sgmFiles.Length > 0)
+ {
+ foreach (FileInfo sgmFile in sgmFiles)
+ {
+ ExtractFile(sgmFile);
+ }
+ }
+ else
+ {
+ SystemConsole.Error.WriteLine("No .sgm files in " + reutersDir);
+ }
+ }
+
+ internal Regex EXTRACTION_PATTERN = new Regex("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>", RegexOptions.Compiled);
+
+ private static string[] META_CHARS = { "&", "<", ">", "\"", "'" };
+
+ private static string[] META_CHARS_SERIALIZATIONS = { "&", "<",
+ ">", """, "'" };
+
+ /// <summary>
+ /// Override if you wish to change what is extracted
+ /// </summary>
+ protected virtual void ExtractFile(FileInfo sgmFile)
+ {
+ try
+ {
+ using (TextReader reader = new StreamReader(new FileStream(sgmFile.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8))
+ {
+ StringBuilder buffer = new StringBuilder(1024);
+ StringBuilder outBuffer = new StringBuilder(1024);
+
+ string line = null;
+ int docNumber = 0;
+ while ((line = reader.ReadLine()) != null)
+ {
+ // when we see a closing reuters tag, flush the file
+
+ if (line.IndexOf("</REUTERS") == -1)
+ {
+ // Replace the SGM escape sequences
+
+ buffer.Append(line).Append(' ');// accumulate the strings for now,
+ // then apply regular expression to
+ // get the pieces,
+ }
+ else
+ {
+ // Extract the relevant pieces and write to a file in the output dir
+ Match matcher = EXTRACTION_PATTERN.Match(buffer.ToString());
+ if (matcher.Success)
+ {
+ do
+ {
+ for (int i = 1; i <= matcher.Groups.Count; i++)
+ {
+ if (matcher.Groups[i] != null)
+ {
+ outBuffer.Append(matcher.Groups[i].Value);
+ }
+ }
+ outBuffer.Append(LINE_SEPARATOR).Append(LINE_SEPARATOR);
+ } while ((matcher = matcher.NextMatch()).Success);
+ }
+
+ string @out = outBuffer.ToString();
+ for (int i = 0; i < META_CHARS_SERIALIZATIONS.Length; i++)
+ {
+ @out = @out.Replace(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
+ }
+ string outFile = System.IO.Path.Combine(outputDir.FullName, sgmFile.Name + "-"
+ + (docNumber++) + ".txt");
+ // System.out.println("Writing " + outFile);
+ StreamWriter writer = new StreamWriter(new FileStream(outFile, FileMode.Create, FileAccess.Write), Encoding.UTF8);
+ writer.Write(@out);
+ writer.Dispose();
+ outBuffer.Length = 0;
+ buffer.Length = 0;
+ }
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+
+ public static void Main(string[] args)
+ {
+ if (args.Length != 2)
+ {
+ Usage("Wrong number of arguments (" + args.Length + ")");
+ return;
+ }
+ DirectoryInfo reutersDir = new DirectoryInfo(args[0]);
+ if (!reutersDir.Exists)
+ {
+ Usage("Cannot find Path to Reuters SGM files (" + reutersDir + ")");
+ return;
+ }
+
+ // First, extract to a tmp directory and only if everything succeeds, rename
+ // to output directory.
+ DirectoryInfo outputDir = new DirectoryInfo(args[1]);
+ outputDir = new DirectoryInfo(outputDir.FullName + "-tmp");
+ outputDir.Create();
+ ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
+ extractor.Extract();
+ // Now rename to requested output dir
+ outputDir.MoveTo(args[1]);
+ }
+
+ private static void Usage(string msg)
+ {
+ SystemConsole.Error.WriteLine("Usage: " + msg + " :: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Utils/ExtractWikipedia.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Utils/ExtractWikipedia.cs b/src/Lucene.Net.Benchmark/Utils/ExtractWikipedia.cs
new file mode 100644
index 0000000..b61fbc5
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Utils/ExtractWikipedia.cs
@@ -0,0 +1,178 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Extract the downloaded Wikipedia dump into separate files for indexing.
+ /// </summary>
+ public class ExtractWikipedia
+ {
+ private DirectoryInfo outputDir;
+
+ public static int count = 0;
+
+ internal static readonly int BASE = 10;
+ protected DocMaker m_docMaker;
+
+ public ExtractWikipedia(DocMaker docMaker, DirectoryInfo outputDir)
+ {
+ this.outputDir = outputDir;
+ this.m_docMaker = docMaker;
+ SystemConsole.WriteLine("Deleting all files in " + outputDir);
+ FileInfo[] files = outputDir.GetFiles();
+ for (int i = 0; i < files.Length; i++)
+ {
+ files[i].Delete();
+ }
+ }
+
+ public virtual DirectoryInfo Directory(int count, DirectoryInfo directory)
+ {
+ if (directory == null)
+ {
+ directory = outputDir;
+ }
+ int @base = BASE;
+ while (@base <= count)
+ {
+ @base *= BASE;
+ }
+ if (count < BASE)
+ {
+ return directory;
+ }
+ directory = new DirectoryInfo(System.IO.Path.Combine(directory.FullName, (((int)(@base / BASE)).ToString(CultureInfo.InvariantCulture))));
+ directory = new DirectoryInfo(System.IO.Path.Combine(directory.FullName, (((int)(count / (@base / BASE))).ToString(CultureInfo.InvariantCulture))));
+ return Directory(count % (@base / BASE), directory);
+ }
+
+ public virtual void Create(string id, string title, string time, string body)
+ {
+ DirectoryInfo d = Directory(count++, null);
+ d.Create();
+ FileInfo f = new FileInfo(System.IO.Path.Combine(d.FullName, id + ".txt"));
+
+ StringBuilder contents = new StringBuilder();
+
+ contents.Append(time);
+ contents.Append("\n\n");
+ contents.Append(title);
+ contents.Append("\n\n");
+ contents.Append(body);
+ contents.Append("\n");
+
+ try
+ {
+ using (TextWriter writer = new StreamWriter(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), Encoding.UTF8))
+ writer.Write(contents.ToString());
+ }
+ catch (IOException ioe)
+ {
+ throw new Exception(ioe.ToString(), ioe);
+ }
+ }
+
+ public virtual void Extract()
+ {
+ Document doc = null;
+ SystemConsole.WriteLine("Starting Extraction");
+ long start = Support.Time.CurrentTimeMilliseconds();
+ try
+ {
+ while ((doc = m_docMaker.MakeDocument()) != null)
+ {
+ Create(doc.Get(DocMaker.ID_FIELD), doc.Get(DocMaker.TITLE_FIELD), doc
+ .Get(DocMaker.DATE_FIELD), doc.Get(DocMaker.BODY_FIELD));
+ }
+ }
+ catch (NoMoreDataException /*e*/)
+ {
+ //continue
+ }
+ long finish = Support.Time.CurrentTimeMilliseconds();
+ SystemConsole.WriteLine("Extraction took " + (finish - start) + " ms");
+ }
+
+ public static void Main(string[] args)
+ {
+
+ FileInfo wikipedia = null;
+ DirectoryInfo outputDir = new DirectoryInfo("./enwiki");
+ bool keepImageOnlyDocs = true;
+ for (int i = 0; i < args.Length; i++)
+ {
+ string arg = args[i];
+ if (arg.Equals("--input", StringComparison.Ordinal) || arg.Equals("-i", StringComparison.Ordinal))
+ {
+ wikipedia = new FileInfo(args[i + 1]);
+ i++;
+ }
+ else if (arg.Equals("--output", StringComparison.Ordinal) || arg.Equals("-o", StringComparison.Ordinal))
+ {
+ outputDir = new DirectoryInfo(args[i + 1]);
+ i++;
+ }
+ else if (arg.Equals("--discardImageOnlyDocs", StringComparison.Ordinal) || arg.Equals("-d", StringComparison.Ordinal))
+ {
+ keepImageOnlyDocs = false;
+ }
+ }
+
+ IDictionary<string, string> properties = new Dictionary<string, string>();
+ properties["docs.file"] = wikipedia.FullName;
+ properties["content.source.forever"] = "false";
+ properties["keep.image.only.docs"] = keepImageOnlyDocs.ToString();
+ Config config = new Config(properties);
+
+ ContentSource source = new EnwikiContentSource();
+ source.SetConfig(config);
+
+ DocMaker docMaker = new DocMaker();
+ docMaker.SetConfig(config, source);
+ docMaker.ResetInputs();
+ if (wikipedia.Exists)
+ {
+ SystemConsole.WriteLine("Extracting Wikipedia to: " + outputDir + " using EnwikiContentSource");
+ outputDir.Create();
+ ExtractWikipedia extractor = new ExtractWikipedia(docMaker, outputDir);
+ extractor.Extract();
+ }
+ else
+ {
+ PrintUsage();
+ }
+ }
+
+ private static void PrintUsage()
+ {
+ SystemConsole.Error.WriteLine("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractWikipedia --input|-i <Path to Wikipedia XML file> " +
+ "[--output|-o <Output Path>] [--discardImageOnlyDocs|-d]");
+ SystemConsole.Error.WriteLine("--discardImageOnlyDocs tells the extractor to skip Wiki docs that contain only images");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/project.json b/src/Lucene.Net.Benchmark/project.json
new file mode 100644
index 0000000..adac6d5
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/project.json
@@ -0,0 +1,53 @@
+{
+ "version": "4.8.0",
+ "title": "Lucene.Net.Benchmark",
+ "description": "System for benchmarking the Lucene.Net full-text search engine library from The Apache Software Foundation.",
+ "authors": [ "The Apache Software Foundation" ],
+ "packOptions": {
+ "projectUrl": "http://lucenenet.apache.org/",
+ "licenseUrl": "https://github.com/apache/lucenenet/blob/master/LICENSE.txt",
+ "iconUrl": "https://github.com/apache/lucenenet/blob/master/branding/logo/lucene-net-icon-128x128.png?raw=true",
+ "owners": [ "The Apache Software Foundation" ],
+ "repository": { "url": "https://github.com/apache/lucenenet" },
+ "tags": [ "lucene.net", "core", "text", "search", "information", "retrieval", "lucene", "apache", "analysis", "index", "query" ]
+ },
+ "buildOptions": {
+ "compile": {
+ "includeFiles": [ "../CommonAssemblyInfo.cs" ]
+ },
+ "nowarn": [ "1591", "1573" ]
+ },
+ "dependencies": {
+ "icu.net": "54.1.1-alpha",
+ "Lucene.Net": "4.8.0",
+ "Lucene.Net.Analysis.Common": "4.8.0",
+ "Lucene.Net.Facet": "4.8.0",
+ "Lucene.Net.Highlighter": "4.8.0",
+ "Lucene.Net.ICU": "4.8.0",
+ "Lucene.Net.Queries": "4.8.0",
+ "Lucene.Net.QueryParser": "4.8.0",
+ "Lucene.Net.Spatial": "4.8.0",
+ "Sax.Net": "2.0.2",
+ "SharpZipLib": "0.86.0",
+ "Spatial4n.Core": "0.4.1-beta00003",
+ "TagSoup.Net": "1.2.1.1"
+ },
+ "frameworks": {
+ "netstandard1.5": {
+ "imports": "dnxcore50",
+ "buildOptions": {
+ "debugType": "portable",
+ "define": [ "NETSTANDARD" ]
+ },
+ "dependencies": {
+ "NETStandard.Library": "1.6.0"
+ }
+ },
+ "net451": {
+ "buildOptions": {
+ "debugType": "full",
+ "define": [ "FEATURE_SERIALIZABLE" ]
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.TestFramework/Util/TestUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.TestFramework/Util/TestUtil.cs b/src/Lucene.Net.TestFramework/Util/TestUtil.cs
index e7eb247..bfc73dd 100644
--- a/src/Lucene.Net.TestFramework/Util/TestUtil.cs
+++ b/src/Lucene.Net.TestFramework/Util/TestUtil.cs
@@ -148,11 +148,20 @@ namespace Lucene.Net.Util
{
foreach (var entry in zip.Entries)
{
+ // Ignore internal folders - these are tacked onto the FullName anyway
+ if (entry.FullName.EndsWith("/", StringComparison.Ordinal) || entry.FullName.EndsWith("\\", StringComparison.Ordinal))
+ {
+ continue;
+ }
using (Stream input = entry.Open())
{
- FileInfo targetFile = new FileInfo(Path.Combine(destDir.FullName, entry.FullName));
+ FileInfo targetFile = new FileInfo(CorrectPath(Path.Combine(destDir.FullName, entry.FullName)));
+ if (!targetFile.Directory.Exists)
+ {
+ targetFile.Directory.Create();
+ }
- using (Stream output = new FileStream(targetFile.FullName, FileMode.OpenOrCreate, FileAccess.Write))
+ using (Stream output = new FileStream(targetFile.FullName, FileMode.Create, FileAccess.Write))
{
input.CopyTo(output);
}
@@ -161,6 +170,15 @@ namespace Lucene.Net.Util
}
}
+ private static string CorrectPath(string input)
+ {
+ if (Path.DirectorySeparatorChar.Equals('/'))
+ {
+ return input.Replace('\\', '/');
+ }
+ return input.Replace('/', '\\');
+ }
+
public static void SyncConcurrentMerges(IndexWriter writer)
{
SyncConcurrentMerges(writer.Config.MergeScheduler);
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/BenchmarkTestCase.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/BenchmarkTestCase.cs b/src/Lucene.Net.Tests.Benchmark/BenchmarkTestCase.cs
new file mode 100644
index 0000000..8981ee0
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/BenchmarkTestCase.cs
@@ -0,0 +1,129 @@
+using Lucene.Net.Benchmarks.ByTask;
+using Lucene.Net.Util;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Base class for all Benchmark unit tests.
+ /// </summary>
+ public abstract class BenchmarkTestCase : LuceneTestCase
+ {
+ private static DirectoryInfo WORKDIR;
+
+ public override void BeforeClass()
+ {
+ base.BeforeClass();
+ WORKDIR = CreateTempDir("benchmark");
+ // LUCENENET: Our directory numbers are sequential. Doing a delete
+ // here will make threads collide.
+ //WORKDIR.Delete();
+ //WORKDIR.Create();
+
+ propLines = new string[] {
+ "work.dir=" + getWorkDirPath(),
+ "directory=RAMDirectory",
+ "print.props=false",
+ };
+ }
+
+ public override void AfterClass()
+ {
+ WORKDIR = null;
+ base.AfterClass();
+ }
+
+
+ public DirectoryInfo getWorkDir()
+ {
+ return WORKDIR;
+ }
+
+ /** Copy a resource into the workdir */
+ public void copyToWorkDir(string resourceName)
+ {
+ Stream resource = GetType().getResourceAsStream(resourceName);
+ Stream dest = new FileStream(System.IO.Path.Combine(getWorkDir().FullName, resourceName), FileMode.Create, FileAccess.Write);
+ byte[] buffer = new byte[8192];
+ int len;
+
+ while ((len = resource.Read(buffer, 0, buffer.Length)) > 0)
+ {
+ dest.Write(buffer, 0, len);
+ }
+
+ resource.Dispose();
+ dest.Dispose();
+ }
+
+ /** Return a path, suitable for a .alg config file, for a resource in the workdir */
+ public String getWorkDirResourcePath(String resourceName)
+ {
+ return System.IO.Path.Combine(getWorkDir().FullName, resourceName).Replace("\\", "/");
+ }
+
+ /** Return a path, suitable for a .alg config file, for the workdir */
+ public String getWorkDirPath()
+ {
+ return getWorkDir().FullName.Replace("\\", "/");
+ }
+
+ // create the benchmark and execute it.
+ public Benchmark execBenchmark(String[] algLines)
+ {
+ String algText = algLinesToText(algLines);
+ logTstLogic(algText);
+ Benchmark benchmark = new Benchmark(new StringReader(algText));
+ benchmark.Execute();
+ return benchmark;
+ }
+
+ // properties in effect in all tests here
+ String[] propLines;
+
+ static readonly String NEW_LINE = Environment.NewLine;
+
+ // catenate alg lines to make the alg text
+ private String algLinesToText(String[] algLines)
+ {
+ String indent = " ";
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < propLines.Length; i++)
+ {
+ sb.append(indent).append(propLines[i]).append(NEW_LINE);
+ }
+ for (int i = 0; i < algLines.Length; i++)
+ {
+ sb.append(indent).append(algLines[i]).append(NEW_LINE);
+ }
+ return sb.toString();
+ }
+
+ private static void logTstLogic(String txt)
+ {
+ if (!VERBOSE)
+ return;
+ Console.WriteLine("Test logic of:");
+ Console.WriteLine(txt);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
new file mode 100644
index 0000000..301c807
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/DocMakerTest.cs
@@ -0,0 +1,193 @@
+using Lucene.Net.Analysis.Core;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using NUnit.Framework;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link DocMaker}.
+ /// </summary>
+ public class DocMakerTest : BenchmarkTestCase
+ {
+ public sealed class OneDocSource : ContentSource
+ {
+ private bool finish = false;
+
+ protected override void Dispose(bool disposing)
+ {
+ }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ if (finish)
+ {
+ throw new NoMoreDataException();
+ }
+
+ docData.Body = ("body");
+ docData.SetDate("date");
+ docData.Title = ("title");
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["key"] = "value";
+ docData.Props = props;
+ finish = true;
+
+ return docData;
+ }
+ }
+
+ private void doTestIndexProperties(bool setIndexProps,
+ bool indexPropsVal, int numExpectedResults)
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+
+ // Indexing configuration.
+ props["analyzer"] = typeof(WhitespaceAnalyzer).AssemblyQualifiedName;
+ props["content.source"] = typeof(OneDocSource).AssemblyQualifiedName;
+ props["directory"] = "RAMDirectory";
+ if (setIndexProps)
+ {
+ props["doc.index.props"] = indexPropsVal.ToString();
+ }
+
+ // Create PerfRunData
+ Config config = new Config(props);
+ PerfRunData runData = new PerfRunData(config);
+
+ TaskSequence tasks = new TaskSequence(runData, TestName, null, false);
+ tasks.AddTask(new CreateIndexTask(runData));
+ tasks.AddTask(new AddDocTask(runData));
+ tasks.AddTask(new CloseIndexTask(runData));
+ tasks.DoLogic();
+
+ IndexReader reader = DirectoryReader.Open(runData.Directory);
+ IndexSearcher searcher = NewSearcher(reader);
+ TopDocs td = searcher.Search(new TermQuery(new Term("key", "value")), 10);
+ assertEquals(numExpectedResults, td.TotalHits);
+ reader.Dispose();
+ }
+
+ private Document createTestNormsDocument(bool setNormsProp,
+ bool normsPropVal, bool setBodyNormsProp, bool bodyNormsVal)
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+
+ // Indexing configuration.
+ props["analyzer"] = typeof(WhitespaceAnalyzer).AssemblyQualifiedName;
+ props["directory"] = "RAMDirectory";
+ if (setNormsProp)
+ {
+ props["doc.tokenized.norms"] = normsPropVal.ToString(CultureInfo.InvariantCulture);
+ }
+ if (setBodyNormsProp)
+ {
+ props["doc.body.tokenized.norms"] = bodyNormsVal.ToString(CultureInfo.InvariantCulture);
+ }
+
+ // Create PerfRunData
+ Config config = new Config(props);
+
+ DocMaker dm = new DocMaker();
+ dm.SetConfig(config, new OneDocSource());
+ return dm.MakeDocument();
+ }
+
+ /* Tests doc.index.props property. */
+ [Test]
+ public void TestIndexProperties()
+ {
+ // default is to not index properties.
+ doTestIndexProperties(false, false, 0);
+
+ // set doc.index.props to false.
+ doTestIndexProperties(true, false, 0);
+
+ // set doc.index.props to true.
+ doTestIndexProperties(true, true, 1);
+ }
+
+ /* Tests doc.tokenized.norms and doc.body.tokenized.norms properties. */
+ [Test]
+ public void TestNorms()
+ {
+
+ Document doc;
+
+ // Don't set anything, use the defaults
+ doc = createTestNormsDocument(false, false, false, false);
+ assertTrue(doc.GetField(DocMaker.TITLE_FIELD).FieldType.OmitNorms);
+ assertFalse(doc.GetField(DocMaker.BODY_FIELD).FieldType.OmitNorms);
+
+ // Set norms to false
+ doc = createTestNormsDocument(true, false, false, false);
+ assertTrue(doc.GetField(DocMaker.TITLE_FIELD).FieldType.OmitNorms);
+ assertFalse(doc.GetField(DocMaker.BODY_FIELD).FieldType.OmitNorms);
+
+ // Set norms to true
+ doc = createTestNormsDocument(true, true, false, false);
+ assertFalse(doc.GetField(DocMaker.TITLE_FIELD).FieldType.OmitNorms);
+ assertFalse(doc.GetField(DocMaker.BODY_FIELD).FieldType.OmitNorms);
+
+ // Set body norms to false
+ doc = createTestNormsDocument(false, false, true, false);
+ assertTrue(doc.GetField(DocMaker.TITLE_FIELD).FieldType.OmitNorms);
+ assertTrue(doc.GetField(DocMaker.BODY_FIELD).FieldType.OmitNorms);
+
+ // Set body norms to true
+ doc = createTestNormsDocument(false, false, true, true);
+ assertTrue(doc.GetField(DocMaker.TITLE_FIELD).FieldType.OmitNorms);
+ assertFalse(doc.GetField(DocMaker.BODY_FIELD).FieldType.OmitNorms);
+ }
+
+ [Test]
+ public void TestDocMakerLeak()
+ {
+ // DocMaker did not close its ContentSource if resetInputs was called twice,
+ // leading to a file handle leak.
+ FileInfo f = new FileInfo(Path.Combine(getWorkDir().FullName, "docMakerLeak.txt"));
+ TextWriter ps = new StreamWriter(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), Encoding.UTF8);
+ ps.WriteLine("one title\t" + Time.CurrentTimeMilliseconds() + "\tsome content");
+ ps.Dispose();
+
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["docs.file"] = f.FullName;
+ props["content.source.forever"] = "false";
+ Config config = new Config(props);
+
+ ContentSource source = new LineDocSource();
+ source.SetConfig(config);
+
+ DocMaker dm = new DocMaker();
+ dm.SetConfig(config, source);
+ dm.ResetInputs();
+ dm.ResetInputs();
+ dm.Dispose();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
new file mode 100644
index 0000000..95ded38
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/EnwikiContentSourceTest.cs
@@ -0,0 +1,194 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ [Ignore("LUCENENET TODO: Never finishes")]
+ public class EnwikiContentSourceTest : LuceneTestCase
+ {
+ /** An EnwikiContentSource which works on a String and not files. */
+ private class StringableEnwikiSource : EnwikiContentSource
+ {
+
+
+ private readonly String docs;
+
+ public StringableEnwikiSource(String docs)
+ {
+ this.docs = docs;
+ }
+
+ protected override Stream OpenInputStream()
+ {
+ return new MemoryStream(Encoding.UTF8.GetBytes(docs));
+ }
+
+ }
+
+ private void assertDocData(DocData dd, String expName, String expTitle, String expBody, String expDate)
+ {
+ assertNotNull(dd);
+ assertEquals(expName, dd.Name);
+ assertEquals(expTitle, dd.Title);
+ assertEquals(expBody, dd.Body);
+ assertEquals(expDate, dd.Date);
+ }
+
+ private void assertNoMoreDataException(EnwikiContentSource stdm)
+ {
+ try
+ {
+ stdm.GetNextDocData(null);
+ fail("Expecting NoMoreDataException");
+ }
+#pragma warning disable 168
+ catch (NoMoreDataException e)
+#pragma warning restore 168
+ {
+ // expected
+ }
+ }
+
+ private readonly String PAGE1 =
+ " <page>\r\n" +
+ " <title>Title1</title>\r\n" +
+ " <ns>0</ns>\r\n" +
+ " <id>1</id>\r\n" +
+ " <revision>\r\n" +
+ " <id>11</id>\r\n" +
+ " <parentid>111</parentid>\r\n" +
+ " <timestamp>2011-09-14T11:35:09Z</timestamp>\r\n" +
+ " <contributor>\r\n" +
+ " <username>Mister1111</username>\r\n" +
+ " <id>1111</id>\r\n" +
+ " </contributor>\r\n" +
+ " <minor />\r\n" +
+ " <comment>/* Never mind */</comment>\r\n" +
+ " <text>Some text 1 here</text>\r\n" +
+ " </revision>\r\n" +
+ " </page>\r\n";
+
+ private readonly String PAGE2 =
+ " <page>\r\n" +
+ " <title>Title2</title>\r\n" +
+ " <ns>0</ns>\r\n" +
+ " <id>2</id>\r\n" +
+ " <revision>\r\n" +
+ " <id>22</id>\r\n" +
+ " <parentid>222</parentid>\r\n" +
+ " <timestamp>2022-09-14T22:35:09Z</timestamp>\r\n" +
+ " <contributor>\r\n" +
+ " <username>Mister2222</username>\r\n" +
+ " <id>2222</id>\r\n" +
+ " </contributor>\r\n" +
+ " <minor />\r\n" +
+ " <comment>/* Never mind */</comment>\r\n" +
+ " <text>Some text 2 here</text>\r\n" +
+ " </revision>\r\n" +
+ " </page>\r\n";
+
+ [Test]
+ public void TestOneDocument()
+ {
+ String docs =
+ "<mediawiki>\r\n" +
+ PAGE1 +
+ "</mediawiki>";
+
+ EnwikiContentSource source = createContentSource(docs, false);
+
+ DocData dd = source.GetNextDocData(new DocData());
+ assertDocData(dd, "1", "Title1", "Some text 1 here", "14-SEP-2011 11:35:09.000");
+
+
+ assertNoMoreDataException(source);
+ }
+
+ private EnwikiContentSource createContentSource(String docs, bool forever)
+ {
+
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["print.props"] = "false";
+ props["content.source.forever"] = forever.ToString(CultureInfo.InvariantCulture);
+ Config config = new Config(props);
+
+ EnwikiContentSource source = new StringableEnwikiSource(docs);
+ source.SetConfig(config);
+
+ // doc-maker just for initiating content source inputs
+ DocMaker docMaker = new DocMaker();
+ docMaker.SetConfig(config, source);
+ docMaker.ResetInputs();
+ return source;
+ }
+
+ [Test]
+ public void TestTwoDocuments()
+ {
+ String docs =
+ "<mediawiki>\r\n" +
+ PAGE1 +
+ PAGE2 +
+ "</mediawiki>";
+
+ EnwikiContentSource source = createContentSource(docs, false);
+
+ DocData dd1 = source.GetNextDocData(new DocData());
+ assertDocData(dd1, "1", "Title1", "Some text 1 here", "14-SEP-2011 11:35:09.000");
+
+ DocData dd2 = source.GetNextDocData(new DocData());
+ assertDocData(dd2, "2", "Title2", "Some text 2 here", "14-SEP-2022 22:35:09.000");
+
+
+ assertNoMoreDataException(source);
+ }
+
+ [Test]
+ public void TestForever()
+ {
+ String docs =
+ "<mediawiki>\r\n" +
+ PAGE1 +
+ PAGE2 +
+ "</mediawiki>";
+
+ EnwikiContentSource source = createContentSource(docs, true);
+
+ // same documents several times
+ for (int i = 0; i < 3; i++)
+ {
+ DocData dd1 = source.GetNextDocData(new DocData());
+ assertDocData(dd1, "1", "Title1", "Some text 1 here", "14-SEP-2011 11:35:09.000");
+
+ DocData dd2 = source.GetNextDocData(new DocData());
+ assertDocData(dd2, "2", "Title2", "Some text 2 here", "14-SEP-2022 22:35:09.000");
+ // Don't test that NoMoreDataException is thrown, since the forever flag is turned on.
+ }
+
+ source.Dispose();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/LineDocSourceTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/LineDocSourceTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/LineDocSourceTest.cs
new file mode 100644
index 0000000..7cd27f1
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/LineDocSourceTest.cs
@@ -0,0 +1,271 @@
+using ICSharpCode.SharpZipLib.BZip2;
+using Lucene.Net.Analysis.Core;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link LineDocSource}.
+ /// </summary>
+ public class LineDocSourceTest : BenchmarkTestCase
+ {
+ //private static final CompressorStreamFactory csFactory = new CompressorStreamFactory();
+
+ private void createBZ2LineFile(FileInfo file, bool addHeader)
+ {
+ Stream @out = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
+ @out = new BZip2OutputStream(@out); // csFactory.createCompressorOutputStream("bzip2", @out);
+ TextWriter writer = new StreamWriter(@out, Encoding.UTF8);
+ writeDocsToFile(writer, addHeader, null);
+ writer.Dispose();
+ }
+
+ private void writeDocsToFile(TextWriter writer, bool addHeader, IDictionary<string, string> otherFields)
+ {
+ if (addHeader)
+ {
+ writer.Write(WriteLineDocTask.FIELDS_HEADER_INDICATOR);
+ writer.Write(WriteLineDocTask.SEP);
+ writer.Write(DocMaker.TITLE_FIELD);
+ writer.Write(WriteLineDocTask.SEP);
+ writer.Write(DocMaker.DATE_FIELD);
+ writer.Write(WriteLineDocTask.SEP);
+ writer.Write(DocMaker.BODY_FIELD);
+ if (otherFields != null)
+ {
+ // additional field names in the header
+ foreach (Object fn in otherFields.Keys)
+ {
+ writer.Write(WriteLineDocTask.SEP);
+ writer.Write(fn.toString());
+ }
+ }
+ writer.WriteLine();
+ }
+ StringBuilder doc = new StringBuilder();
+ doc.append("title").append(WriteLineDocTask.SEP).append("date").append(WriteLineDocTask.SEP).append(DocMaker.BODY_FIELD);
+ if (otherFields != null)
+ {
+ // additional field values in the doc line
+ foreach (Object fv in otherFields.Values)
+ {
+ doc.append(WriteLineDocTask.SEP).append(fv.toString());
+ }
+ }
+ writer.Write(doc.toString());
+ writer.WriteLine();
+ }
+
+ private void createRegularLineFile(FileInfo file, bool addHeader)
+ {
+ Stream @out = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
+ TextWriter writer = new StreamWriter(@out, Encoding.UTF8);
+ writeDocsToFile(writer, addHeader, null);
+ writer.Dispose();
+ }
+
+ private void createRegularLineFileWithMoreFields(FileInfo file, params String[] extraFields)
+ {
+ Stream @out = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
+ TextWriter writer = new StreamWriter(@out, Encoding.UTF8);
+ Dictionary<string, string> p = new Dictionary<string, string>();
+ foreach (String f in extraFields)
+ {
+ p[f] = f;
+ }
+ writeDocsToFile(writer, true, p);
+ writer.Dispose();
+ }
+
+ private void doIndexAndSearchTest(FileInfo file, Type lineParserClass, String storedField)
+ {
+ doIndexAndSearchTestWithRepeats(file, lineParserClass, 1, storedField); // no extra repetitions
+ doIndexAndSearchTestWithRepeats(file, lineParserClass, 2, storedField); // 1 extra repetition
+ doIndexAndSearchTestWithRepeats(file, lineParserClass, 4, storedField); // 3 extra repetitions
+ }
+
+ private void doIndexAndSearchTestWithRepeats(FileInfo file,
+ Type lineParserClass, int numAdds, String storedField)
+ {
+
+ IndexReader reader = null;
+ IndexSearcher searcher = null;
+ PerfRunData runData = null;
+ try
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+
+ // LineDocSource specific settings.
+ props["docs.file"] = file.FullName;
+ if (lineParserClass != null)
+ {
+ props["line.parser"] = lineParserClass.AssemblyQualifiedName;
+ }
+
+ // Indexing configuration.
+ props["analyzer"] = typeof(WhitespaceAnalyzer).AssemblyQualifiedName;
+ props["content.source"] = typeof(LineDocSource).AssemblyQualifiedName;
+ props["directory"] = "RAMDirectory";
+ props["doc.stored"] = "true";
+ props["doc.index.props"] = "true";
+
+ // Create PerfRunData
+ Config config = new Config(props);
+ runData = new PerfRunData(config);
+
+ TaskSequence tasks = new TaskSequence(runData, "testBzip2", null, false);
+ tasks.AddTask(new CreateIndexTask(runData));
+ for (int i = 0; i < numAdds; i++)
+ {
+ tasks.AddTask(new AddDocTask(runData));
+ }
+ tasks.AddTask(new CloseIndexTask(runData));
+ try
+ {
+ tasks.DoLogic();
+ }
+ finally
+ {
+ tasks.Dispose();
+ }
+
+ reader = DirectoryReader.Open(runData.Directory);
+ searcher = NewSearcher(reader);
+ TopDocs td = searcher.Search(new TermQuery(new Term("body", "body")), 10);
+ assertEquals(numAdds, td.TotalHits);
+ assertNotNull(td.ScoreDocs[0]);
+
+ if (storedField == null)
+ {
+ storedField = DocMaker.BODY_FIELD; // added to all docs and satisfies field-name == value
+ }
+ assertEquals("Wrong field value", storedField, searcher.Doc(0).Get(storedField));
+ }
+ finally
+ {
+ IOUtils.Dispose(reader, runData);
+ }
+
+ }
+
+ /* Tests LineDocSource with a bzip2 input stream. */
+ [Test]
+ public void TestBZip2()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line.bz2"));
+ createBZ2LineFile(file, true);
+ doIndexAndSearchTest(file, null, null);
+ }
+
+ [Test]
+ public void TestBZip2NoHeaderLine()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line.bz2"));
+ createBZ2LineFile(file, false);
+ doIndexAndSearchTest(file, null, null);
+ }
+
+ [Test]
+ public void TestRegularFile()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ createRegularLineFile(file, true);
+ doIndexAndSearchTest(file, null, null);
+ }
+
+ [Test]
+ public void TestRegularFileSpecialHeader()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ createRegularLineFile(file, true);
+ doIndexAndSearchTest(file, typeof(HeaderLineParser), null);
+ }
+
+ [Test]
+ public void TestRegularFileNoHeaderLine()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ createRegularLineFile(file, false);
+ doIndexAndSearchTest(file, null, null);
+ }
+
+ [Test]
+ public void TestInvalidFormat()
+ {
+ String[]
+ testCases = new String[] {
+ "", // empty line
+ "title", // just title
+ "title" + WriteLineDocTask.SEP, // title + SEP
+ "title" + WriteLineDocTask.SEP + "body", // title + SEP + body
+ // note that title + SEP + body + SEP is a valid line, which results in an
+ // empty body
+ };
+
+ for (int i = 0; i < testCases.Length; i++)
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ TextWriter writer = new StreamWriter(new FileStream(file.FullName, FileMode.Create, FileAccess.Write), Encoding.UTF8);
+ writer.Write(testCases[i]);
+ writer.WriteLine();
+ writer.Dispose();
+ try
+ {
+ doIndexAndSearchTest(file, null, null);
+ fail("Some exception should have been thrown for: [" + testCases[i] + "]");
+ }
+#pragma warning disable 168
+ catch (Exception e)
+#pragma warning restore 168
+ {
+ // expected.
+ }
+ }
+ }
+
+ /** Doc Name is not part of the default header */
+ [Test]
+ public void TestWithDocsName()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ createRegularLineFileWithMoreFields(file, DocMaker.NAME_FIELD);
+ doIndexAndSearchTest(file, null, DocMaker.NAME_FIELD);
+ }
+
+ /** Use fields names that are not defined in Docmaker and so will go to Properties */
+ [Test]
+ public void TestWithProperties()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ String specialField = "mySpecialField";
+ createRegularLineFileWithMoreFields(file, specialField);
+ doIndexAndSearchTest(file, null, specialField);
+ }
+ }
+}
[02/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs
new file mode 100644
index 0000000..61e7bf1
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs
@@ -0,0 +1,1177 @@
+using Icu.Collation;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.TokenAttributes;
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Collation;
+using Lucene.Net.Facet.Taxonomy;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test very simply that perf tasks - simple algorithms - are doing what they should.
+ /// </summary>
+ [SuppressCodecs("Lucene3x")]
+ public class TestPerfTasksLogic : BenchmarkTestCase
+ {
+ //public override void SetUp()
+ //{
+ // base.SetUp();
+ // copyToWorkDir("reuters.first20.lines.txt");
+ // copyToWorkDir("test-mapping-ISOLatin1Accent-partial.txt");
+ //}
+
+ public override void BeforeClass()
+ {
+ base.BeforeClass();
+ copyToWorkDir("reuters.first20.lines.txt");
+ copyToWorkDir("test-mapping-ISOLatin1Accent-partial.txt");
+ }
+
+ /**
+ * Test index creation logic
+ */
+ [Test]
+ public void TestIndexAndSearchTasks()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : 1000",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{ CountingSearchTest } : 200",
+ "CloseReader",
+ "[ CountingSearchTest > : 70",
+ "[ CountingSearchTest > : 9",
+ };
+
+ // 2. we test this value later
+ CountingSearchTestTask.numSearches = 0;
+
+ // 3. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 4. test specific checks after the benchmark run completed.
+ assertEquals("TestSearchTask was supposed to be called!", 279, CountingSearchTestTask.numSearches);
+ assertTrue("Index does not exist?...!", DirectoryReader.IndexExists(benchmark.RunData.Directory));
+ // now we should be able to open the index for write.
+ IndexWriter iw = new IndexWriter(benchmark.RunData.Directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
+ .SetOpenMode(OpenMode.APPEND));
+ iw.Dispose();
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ assertEquals("1000 docs were added to the index, this is what we expect to find!", 1000, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ /**
+ * Test timed sequence task.
+ */
+ [Test]
+ public void TestTimedSearchTask()
+ {
+ String[] algLines = {
+ "log.step=100000",
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : 100",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{ CountingSearchTest } : .5s",
+ "CloseReader",
+ };
+
+ CountingSearchTestTask.numSearches = 0;
+ execBenchmark(algLines);
+ assertTrue(CountingSearchTestTask.numSearches > 0);
+ long elapsed = CountingSearchTestTask.prevLastMillis - CountingSearchTestTask.startMillis;
+ assertTrue("elapsed time was " + elapsed + " msec", elapsed <= 1500);
+ }
+
+ // disabled until we fix BG thread prio -- this test
+ // causes build to hang
+ [Test]
+ public void TestBGSearchTaskThreads()
+ {
+ String[] algLines = {
+ "log.time.step.msec = 100",
+ "log.step=100000",
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : 1000",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{",
+ " [ \"XSearch\" { CountingSearchTest > : * ] : 2 &-1",
+ " Wait(0.5)",
+ "}",
+ "CloseReader",
+ "RepSumByPref X"
+ };
+
+ CountingSearchTestTask.numSearches = 0;
+ execBenchmark(algLines);
+
+ // NOTE: cannot assert this, because on a super-slow
+ // system, it could be after waiting 0.5 seconds that
+ // the search threads hadn't yet succeeded in starting
+ // up and then they start up and do no searching:
+ //assertTrue(CountingSearchTestTask.numSearches > 0);
+ }
+
+ [Test]
+ public void TestHighlighting()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "doc.stored=true",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "query.maker=" + typeof(ReutersQueryMaker).AssemblyQualifiedName,
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : 100",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{ CountingHighlighterTest(size[1],highlight[1],mergeContiguous[true],maxFrags[1],fields[body]) } : 200",
+ "CloseReader",
+ };
+
+ // 2. we test this value later
+ CountingHighlighterTestTask.numHighlightedResults = 0;
+ CountingHighlighterTestTask.numDocsRetrieved = 0;
+ // 3. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 4. test specific checks after the benchmark run completed.
+ assertEquals("TestSearchTask was supposed to be called!", 92, CountingHighlighterTestTask.numDocsRetrieved);
+ //pretty hard to figure out a priori how many docs are going to have highlighted fragments returned, but we can never have more than the number of docs
+ //we probably should use a different doc/query maker, but...
+ assertTrue("TestSearchTask was supposed to be called!", CountingHighlighterTestTask.numDocsRetrieved >= CountingHighlighterTestTask.numHighlightedResults && CountingHighlighterTestTask.numHighlightedResults > 0);
+
+ assertTrue("Index does not exist?...!", DirectoryReader.IndexExists(benchmark.RunData.Directory));
+ // now we should be able to open the index for write.
+ IndexWriter iw = new IndexWriter(benchmark.RunData.Directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND));
+ iw.Dispose();
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ assertEquals("100 docs were added to the index, this is what we expect to find!", 100, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ [Test]
+ public void TestHighlightingTV()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "doc.stored=true",//doc storage is required in order to have text to highlight
+ "doc.term.vector=true",
+ "doc.term.vector.offsets=true",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "query.maker=" + typeof(ReutersQueryMaker).AssemblyQualifiedName,
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : 1000",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{ CountingHighlighterTest(size[1],highlight[1],mergeContiguous[true],maxFrags[1],fields[body]) } : 200",
+ "CloseReader",
+ };
+
+ // 2. we test this value later
+ CountingHighlighterTestTask.numHighlightedResults = 0;
+ CountingHighlighterTestTask.numDocsRetrieved = 0;
+ // 3. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 4. test specific checks after the benchmark run completed.
+ assertEquals("TestSearchTask was supposed to be called!", 92, CountingHighlighterTestTask.numDocsRetrieved);
+ //pretty hard to figure out a priori how many docs are going to have highlighted fragments returned, but we can never have more than the number of docs
+ //we probably should use a different doc/query maker, but...
+ assertTrue("TestSearchTask was supposed to be called!", CountingHighlighterTestTask.numDocsRetrieved >= CountingHighlighterTestTask.numHighlightedResults && CountingHighlighterTestTask.numHighlightedResults > 0);
+
+ assertTrue("Index does not exist?...!", DirectoryReader.IndexExists(benchmark.RunData.Directory));
+ // now we should be able to open the index for write.
+ IndexWriter iw = new IndexWriter(benchmark.RunData.Directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND));
+ iw.Dispose();
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ assertEquals("1000 docs were added to the index, this is what we expect to find!", 1000, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ [Test]
+ public void TestHighlightingNoTvNoStore()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "doc.stored=false",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "query.maker=" + typeof(ReutersQueryMaker).AssemblyQualifiedName,
+ "ResetSystemErase",
+ "CreateIndex",
+ "{ AddDoc } : 1000",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{ CountingHighlighterTest(size[1],highlight[1],mergeContiguous[true],maxFrags[1],fields[body]) } : 200",
+ "CloseReader",
+ };
+
+ // 2. we test this value later
+ CountingHighlighterTestTask.numHighlightedResults = 0;
+ CountingHighlighterTestTask.numDocsRetrieved = 0;
+ // 3. execute the algorithm (required in every "logic" test)
+ try
+ {
+ Benchmark benchmark = execBenchmark(algLines);
+ assertTrue("CountingHighlighterTest should have thrown an exception", false);
+ assertNotNull(benchmark); // (avoid compile warning on unused variable)
+ }
+#pragma warning disable 168
+ catch (Exception e)
+#pragma warning restore 168
+ {
+ assertTrue(true);
+ }
+ }
+
+ /**
+ * Test Exhasting Doc Maker logic
+ */
+ [Test]
+ public void TestExhaustContentSource()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.SingleDocSource, Lucene.Net.Benchmark",
+ "content.source.log.step=1",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "# ----- alg ",
+ "CreateIndex",
+ "{ AddDoc } : * ",
+ "ForceMerge(1)",
+ "CloseIndex",
+ "OpenReader",
+ "{ CountingSearchTest } : 100",
+ "CloseReader",
+ "[ CountingSearchTest > : 30",
+ "[ CountingSearchTest > : 9",
+ };
+
+ // 2. we test this value later
+ CountingSearchTestTask.numSearches = 0;
+
+ // 3. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 4. test specific checks after the benchmark run completed.
+ assertEquals("TestSearchTask was supposed to be called!", 139, CountingSearchTestTask.numSearches);
+ assertTrue("Index does not exist?...!", DirectoryReader.IndexExists(benchmark.RunData.Directory));
+ // now we should be able to open the index for write.
+ IndexWriter iw = new IndexWriter(benchmark.RunData.Directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND));
+ iw.Dispose();
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ assertEquals("1 docs were added to the index, this is what we expect to find!", 1, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ // LUCENE-1994: test thread safety of SortableSingleDocMaker
+ [Test]
+ public void TestDocMakerThreadSafety()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.SortableSingleDocSource, Lucene.Net.Benchmark",
+ "doc.term.vector=false",
+ "log.step.AddDoc=10000",
+ "content.source.forever=true",
+ "directory=RAMDirectory",
+ "doc.reuse.fields=false",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "doc.index.props=true",
+ "# ----- alg ",
+ "CreateIndex",
+ "[ { AddDoc > : 250 ] : 4",
+ "CloseIndex",
+ };
+
+ // 2. we test this value later
+ CountingSearchTestTask.numSearches = 0;
+
+ // 3. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ DirectoryReader r = DirectoryReader.Open(benchmark.RunData.Directory);
+ SortedDocValues idx = FieldCache.DEFAULT.GetTermsIndex(SlowCompositeReaderWrapper.Wrap(r), "country");
+ int maxDoc = r.MaxDoc;
+ assertEquals(1000, maxDoc);
+ for (int i = 0; i < 1000; i++)
+ {
+ assertTrue("doc " + i + " has null country", idx.GetOrd(i) != -1);
+ }
+ r.Dispose();
+ }
+
+ /**
+ * Test Parallel Doc Maker logic (for LUCENE-940)
+ */
+ [Test]
+ public void TestParallelDocMaker()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=FSDirectory",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "# ----- alg ",
+ "CreateIndex",
+ "[ { AddDoc } : * ] : 4 ",
+ "CloseIndex",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ /**
+ * Test WriteLineDoc and LineDocSource.
+ */
+ [Test]
+ public void TestLineDocFile()
+ {
+ FileInfo lineFile = CreateTempFile("test.reuters.lines", ".txt");
+
+ // We will call WriteLineDocs this many times
+ int NUM_TRY_DOCS = 50;
+
+ // Creates a line file with first 50 docs from SingleDocSource
+ String[] algLines1 = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.SingleDocSource, Lucene.Net.Benchmark",
+ "content.source.forever=true",
+ "line.file.out=" + lineFile.FullName.Replace('\\', '/'),
+ "# ----- alg ",
+ "{WriteLineDoc()}:" + NUM_TRY_DOCS,
+ };
+
+ // Run algo
+ Benchmark benchmark = execBenchmark(algLines1);
+
+ TextReader r =
+ new StreamReader(
+ new FileStream(lineFile.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8);
+ int numLines = 0;
+ String line;
+ while ((line = r.ReadLine()) != null)
+ {
+ if (numLines == 0 && line.StartsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR, StringComparison.Ordinal))
+ {
+ continue; // do not count the header line as a doc
+ }
+ numLines++;
+ }
+ r.Dispose();
+ assertEquals("did not see the right number of docs; should be " + NUM_TRY_DOCS + " but was " + numLines, NUM_TRY_DOCS, numLines);
+
+ // Index the line docs
+ String[] algLines2 = {
+ "# ----- properties ",
+ "analyzer=Lucene.Net.Analysis.Core.WhitespaceAnalyzer, Lucene.Net.Analysis.Common",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + lineFile.FullName.Replace('\\', '/'),
+ "content.source.forever=false",
+ "doc.reuse.fields=false",
+ "ram.flush.mb=4",
+ "# ----- alg ",
+ "ResetSystemErase",
+ "CreateIndex",
+ "{AddDoc}: *",
+ "CloseIndex",
+ };
+
+ // Run algo
+ benchmark = execBenchmark(algLines2);
+
+ // now we should be able to open the index for write.
+ IndexWriter iw = new IndexWriter(benchmark.RunData.Directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
+ .SetOpenMode(OpenMode.APPEND));
+ iw.Dispose();
+
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ assertEquals(numLines + " lines were created but " + ir.NumDocs + " docs are in the index", numLines, ir.NumDocs);
+ ir.Dispose();
+
+ lineFile.Delete();
+ }
+
+ /**
+ * Test ReadTokensTask
+ */
+ [Test]
+ public void TestReadTokens()
+ {
+
+ // We will call ReadTokens on this many docs
+ int NUM_DOCS = 20;
+
+ // Read tokens from first NUM_DOCS docs from Reuters and
+ // then build index from the same docs
+ String[] algLines1 = {
+ "# ----- properties ",
+ "analyzer=Lucene.Net.Analysis.Core.WhitespaceAnalyzer, Lucene.Net.Analysis.Common",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "# ----- alg ",
+ "{ReadTokens}: " + NUM_DOCS,
+ "ResetSystemErase",
+ "CreateIndex",
+ "{AddDoc}: " + NUM_DOCS,
+ "CloseIndex",
+ };
+
+ // Run algo
+ Benchmark benchmark = execBenchmark(algLines1);
+
+ IList<TaskStats> stats = benchmark.RunData.Points.TaskStats;
+
+ // Count how many tokens all ReadTokens saw
+ int totalTokenCount1 = 0;
+ foreach (TaskStats stat in stats)
+ {
+ if (stat.Task.GetName().Equals("ReadTokens"))
+ {
+ totalTokenCount1 += stat.Count;
+ }
+ }
+
+ // Separately count how many tokens are actually in the index:
+ IndexReader reader = DirectoryReader.Open(benchmark.RunData.Directory);
+ assertEquals(NUM_DOCS, reader.NumDocs);
+
+ int totalTokenCount2 = 0;
+
+ Fields fields = MultiFields.GetFields(reader);
+
+ foreach (String fieldName in fields)
+ {
+ if (fieldName.Equals(DocMaker.ID_FIELD) || fieldName.Equals(DocMaker.DATE_MSEC_FIELD) || fieldName.Equals(DocMaker.TIME_SEC_FIELD))
+ {
+ continue;
+ }
+ Terms terms = fields.GetTerms(fieldName);
+ if (terms == null)
+ {
+ continue;
+ }
+ TermsEnum termsEnum = terms.GetIterator(null);
+ DocsEnum docs = null;
+ while (termsEnum.Next() != null)
+ {
+ docs = TestUtil.Docs(Random(), termsEnum, MultiFields.GetLiveDocs(reader), docs, DocsFlags.FREQS);
+ while (docs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+ {
+ totalTokenCount2 += docs.Freq;
+ }
+ }
+ }
+ reader.Dispose();
+
+ // Make sure they are the same
+ assertEquals(totalTokenCount1, totalTokenCount2);
+ }
+
+ /**
+ * Test that " {[AddDoc(4000)]: 4} : * " works corrcetly (for LUCENE-941)
+ */
+ [Test]
+ public void TestParallelExhausted()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "task.max.depth.log=1",
+ "# ----- alg ",
+ "CreateIndex",
+ "{ [ AddDoc]: 4} : * ",
+ "ResetInputs ",
+ "{ [ AddDoc]: 4} : * ",
+ "WaitForMerges",
+ "CloseIndex",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 2 * 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+ }
+
+
+ /**
+ * Test that exhaust in loop works as expected (LUCENE-1115).
+ */
+ [Test]
+ public void TestExhaustedLooped()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "task.max.depth.log=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " WaitForMerges",
+ " CloseIndex",
+ "} : 2",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ /**
+ * Test that we can close IndexWriter with argument "false".
+ */
+ [Test]
+ public void TestCloseIndexFalse()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "ram.flush.mb=-1",
+ "max.buffered=2",
+ "content.source.log.step=3",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "debug.level=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " CloseIndex(false)",
+ "} : 2",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ public class MyMergeScheduler : SerialMergeScheduler
+ {
+ internal bool called;
+ public MyMergeScheduler()
+ : base()
+ {
+ called = true;
+ }
+ }
+
+ /**
+ * Test that we can set merge scheduler".
+ */
+ [Test]
+ public void TestMergeScheduler()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "merge.scheduler=" + typeof(MyMergeScheduler).AssemblyQualifiedName,
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "debug.level=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ "} : 2",
+ };
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ assertTrue("did not use the specified MergeScheduler",
+ ((MyMergeScheduler)benchmark.RunData.IndexWriter.Config
+ .MergeScheduler).called);
+ benchmark.RunData.IndexWriter.Dispose();
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ public class MyMergePolicy : LogDocMergePolicy
+ {
+ internal bool called;
+ public MyMergePolicy()
+ {
+ called = true;
+ }
+ }
+
+ /**
+ * Test that we can set merge policy".
+ */
+ [Test]
+ public void TestMergePolicy()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "ram.flush.mb=-1",
+ "max.buffered=2",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "merge.policy=" + typeof(MyMergePolicy).AssemblyQualifiedName,
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "debug.level=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ "} : 2",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+ assertTrue("did not use the specified MergePolicy", ((MyMergePolicy)benchmark.RunData.IndexWriter.Config.MergePolicy).called);
+ benchmark.RunData.IndexWriter.Dispose();
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+ }
+
+ /**
+ * Test that IndexWriter settings stick.
+ */
+ [Test]
+ public void TestIndexWriterSettings()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "ram.flush.mb=-1",
+ "max.buffered=2",
+ "compound=cmpnd:true:false",
+ "doc.term.vector=vector:false:true",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "merge.factor=3",
+ "doc.tokenized=false",
+ "debug.level=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " NewRound",
+ "} : 2",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+ IndexWriter writer = benchmark.RunData.IndexWriter;
+ assertEquals(2, writer.Config.MaxBufferedDocs);
+ assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, (int)writer.Config.RAMBufferSizeMB);
+ assertEquals(3, ((LogMergePolicy)writer.Config.MergePolicy).MergeFactor);
+ assertEquals(0.0d, writer.Config.MergePolicy.NoCFSRatio, 0.0);
+ writer.Dispose();
+ Store.Directory dir = benchmark.RunData.Directory;
+ IndexReader reader = DirectoryReader.Open(dir);
+ Fields tfv = reader.GetTermVectors(0);
+ assertNotNull(tfv);
+ assertTrue(tfv.Count > 0);
+ reader.Dispose();
+ }
+
+ /**
+ * Test indexing with facets tasks.
+ */
+ [Test]
+ public void TestIndexingWithFacets()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=100",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "merge.factor=3",
+ "doc.tokenized=false",
+ "debug.level=1",
+ "# ----- alg ",
+ "ResetSystemErase",
+ "CreateIndex",
+ "CreateTaxonomyIndex",
+ "{ \"AddDocs\" AddFacetedDoc > : * ",
+ "CloseIndex",
+ "CloseTaxonomyIndex",
+ "OpenTaxonomyReader",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+ PerfRunData runData = benchmark.RunData;
+ assertNull("taxo writer was not properly closed", runData.TaxonomyWriter);
+ TaxonomyReader taxoReader = runData.GetTaxonomyReader();
+ assertNotNull("taxo reader was not opened", taxoReader);
+ assertTrue("nothing was added to the taxnomy (expecting root and at least one addtional category)", taxoReader.Count > 1);
+ taxoReader.Dispose();
+ }
+
+ /**
+ * Test that we can call forceMerge(maxNumSegments).
+ */
+ [Test]
+ public void TestForceMerge()
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "ram.flush.mb=-1",
+ "max.buffered=3",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "merge.policy=Lucene.Net.Index.LogDocMergePolicy, Lucene.Net",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "debug.level=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " ForceMerge(3)",
+ " CloseIndex()",
+ "} : 2",
+ };
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 3. test number of docs in the index
+ IndexReader ir = DirectoryReader.Open(benchmark.RunData.Directory);
+ int ndocsExpected = 20; // first 20 reuters docs.
+ assertEquals("wrong number of docs in the index!", ndocsExpected, ir.NumDocs);
+ ir.Dispose();
+
+ // Make sure we have 3 segments:
+ SegmentInfos infos = new SegmentInfos();
+ infos.Read(benchmark.RunData.Directory);
+ assertEquals(3, infos.Count);
+ }
+
+ /**
+ * Test disabling task count (LUCENE-1136).
+ */
+ [Test]
+ public void TestDisableCounting()
+ {
+ doTestDisableCounting(true);
+ doTestDisableCounting(false);
+ }
+
+ private void doTestDisableCounting(bool disable)
+ {
+ // 1. alg definition (required in every "logic" test)
+ String[] algLines = disableCountingLines(disable);
+
+ // 2. execute the algorithm (required in every "logic" test)
+ Benchmark benchmark = execBenchmark(algLines);
+
+ // 3. test counters
+ int n = disable ? 0 : 1;
+ int nChecked = 0;
+ foreach (TaskStats stats in benchmark.RunData.Points.TaskStats)
+ {
+ String taskName = stats.Task.GetName();
+ if (taskName.equals("Rounds"))
+ {
+ assertEquals("Wrong total count!", 20 + 2 * n, stats.Count);
+ nChecked++;
+ }
+ else if (taskName.equals("CreateIndex"))
+ {
+ assertEquals("Wrong count for CreateIndex!", n, stats.Count);
+ nChecked++;
+ }
+ else if (taskName.equals("CloseIndex"))
+ {
+ assertEquals("Wrong count for CloseIndex!", n, stats.Count);
+ nChecked++;
+ }
+ }
+ assertEquals("Missing some tasks to check!", 3, nChecked);
+ }
+
+ private String[] disableCountingLines(bool disable)
+ {
+ String dis = disable ? "-" : "";
+ return new String[] {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=30",
+ "doc.term.vector=false",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "doc.stored=false",
+ "doc.tokenized=false",
+ "task.max.depth.log=1",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " "+dis+"CreateIndex", // optionally disable counting here
+ " { \"AddDocs\" AddDoc > : * ",
+ " "+dis+" CloseIndex", // optionally disable counting here (with extra blanks)
+ "}",
+ "RepSumByName",
+ };
+ }
+
+ /**
+ * Test that we can change the Locale in the runData,
+ * that it is parsed as we expect.
+ */
+ [Test]
+ public void TestLocale()
+ {
+ // empty Locale: clear it (null)
+ Benchmark benchmark = execBenchmark(getLocaleConfig(""));
+ assertNull(benchmark.RunData.Locale);
+
+ // ROOT locale
+ benchmark = execBenchmark(getLocaleConfig("ROOT"));
+ assertEquals(CultureInfo.InvariantCulture, benchmark.RunData.Locale);
+
+ // specify just a language
+ benchmark = execBenchmark(getLocaleConfig("de"));
+ assertEquals(new CultureInfo("de"), benchmark.RunData.Locale);
+
+ // specify language + country
+ benchmark = execBenchmark(getLocaleConfig("en,US"));
+ assertEquals(new CultureInfo("en-US"), benchmark.RunData.Locale);
+
+ // specify language + country + variant
+ benchmark = execBenchmark(getLocaleConfig("no,NO,NY"));
+ assertEquals(new CultureInfo("no-NO"/*, "NY"*/), benchmark.RunData.Locale);
+ }
+
+ private String[] getLocaleConfig(String localeParam)
+ {
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " NewLocale(" + localeParam + ")",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " NewRound",
+ "} : 1",
+ };
+ return algLines;
+ }
+
+ /**
+ * Test that we can create CollationAnalyzers.
+ */
+ [Test]
+ public void TestCollator()
+ {
+ // LUCENENET specific - we don't have a JDK version of collator
+ // so we are using ICU
+ var collatorParam = "impl:icu";
+
+ // ROOT locale
+ Benchmark benchmark = execBenchmark(getCollatorConfig("ROOT", collatorParam));
+ ICUCollationKeyAnalyzer expected = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator
+ .Create(CultureInfo.InvariantCulture));
+ assertEqualCollation(expected, benchmark.RunData.Analyzer, "foobar");
+
+ // specify just a language
+ benchmark = execBenchmark(getCollatorConfig("de", collatorParam));
+ expected = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.Create(new CultureInfo("de")));
+ assertEqualCollation(expected, benchmark.RunData.Analyzer, "foobar");
+
+ // specify language + country
+ benchmark = execBenchmark(getCollatorConfig("en,US", collatorParam));
+ expected = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.Create(new CultureInfo("en-US"), Collator.Fallback.FallbackAllowed));
+ assertEqualCollation(expected, benchmark.RunData.Analyzer, "foobar");
+
+ // specify language + country + variant
+ benchmark = execBenchmark(getCollatorConfig("no,NO,NY", collatorParam));
+ expected = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.Create(new CultureInfo("no-NO"/*, "NY"*/), Collator.Fallback.FallbackAllowed));
+ assertEqualCollation(expected, benchmark.RunData.Analyzer, "foobar");
+ }
+
+ private void assertEqualCollation(Analyzer a1, Analyzer a2, String text)
+ {
+ TokenStream ts1 = a1.GetTokenStream("bogus", text);
+ TokenStream ts2 = a2.GetTokenStream("bogus", text);
+ ts1.Reset();
+ ts2.Reset();
+ ITermToBytesRefAttribute termAtt1 = ts1.AddAttribute<ITermToBytesRefAttribute>();
+ ITermToBytesRefAttribute termAtt2 = ts2.AddAttribute<ITermToBytesRefAttribute>();
+ assertTrue(ts1.IncrementToken());
+ assertTrue(ts2.IncrementToken());
+ BytesRef bytes1 = termAtt1.BytesRef;
+ BytesRef bytes2 = termAtt2.BytesRef;
+ termAtt1.FillBytesRef();
+ termAtt2.FillBytesRef();
+ assertEquals(bytes1, bytes2);
+ assertFalse(ts1.IncrementToken());
+ assertFalse(ts2.IncrementToken());
+ ts1.Dispose();
+ ts2.Dispose();
+ }
+
+ private String[] getCollatorConfig(String localeParam,
+ String collationParam)
+ {
+ String[] algLines = {
+ "# ----- properties ",
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " NewLocale(" + localeParam + ")",
+ " NewCollationAnalyzer(" + collationParam + ")",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " NewRound",
+ "} : 1",
+ };
+ return algLines;
+ }
+
+ /**
+ * Test that we can create shingle analyzers using AnalyzerFactory.
+ */
+ [Test]
+ public void TestShingleAnalyzer()
+ {
+ String text = "one,two,three, four five six";
+
+ // StandardTokenizer, maxShingleSize, and outputUnigrams
+ Benchmark benchmark = execBenchmark(getAnalyzerFactoryConfig
+ ("shingle-analyzer", "StandardTokenizer,ShingleFilter"));
+ benchmark.RunData.Analyzer.GetTokenStream
+ ("bogus", text).Dispose();
+ BaseTokenStreamTestCase.AssertAnalyzesTo(benchmark.RunData.Analyzer, text,
+ new String[] { "one", "one two", "two", "two three",
+ "three", "three four", "four", "four five",
+ "five", "five six", "six" });
+ // StandardTokenizer, maxShingleSize = 3, and outputUnigrams = false
+ benchmark = execBenchmark
+ (getAnalyzerFactoryConfig
+ ("shingle-analyzer",
+ "StandardTokenizer,ShingleFilter(maxShingleSize:3,outputUnigrams:false)"));
+ BaseTokenStreamTestCase.AssertAnalyzesTo(benchmark.RunData.Analyzer, text,
+ new String[] { "one two", "one two three", "two three",
+ "two three four", "three four",
+ "three four five", "four five",
+ "four five six", "five six" });
+ // WhitespaceTokenizer, default maxShingleSize and outputUnigrams
+ benchmark = execBenchmark
+ (getAnalyzerFactoryConfig("shingle-analyzer", "WhitespaceTokenizer,ShingleFilter"));
+ BaseTokenStreamTestCase.AssertAnalyzesTo(benchmark.RunData.Analyzer, text,
+ new String[] { "one,two,three,", "one,two,three, four",
+ "four", "four five", "five", "five six",
+ "six" });
+
+ // WhitespaceTokenizer, maxShingleSize=3 and outputUnigrams=false
+ benchmark = execBenchmark
+ (getAnalyzerFactoryConfig
+ ("shingle-factory",
+ "WhitespaceTokenizer,ShingleFilter(outputUnigrams:false,maxShingleSize:3)"));
+ BaseTokenStreamTestCase.AssertAnalyzesTo(benchmark.RunData.Analyzer, text,
+ new String[] { "one,two,three, four",
+ "one,two,three, four five",
+ "four five", "four five six",
+ "five six" });
+ }
+
+ private String[] getAnalyzerFactoryConfig(String name, String @params)
+ {
+ //String singleQuoteEscapedName = name.Replace("'", "\\\\'");
+ //String[] algLines = {
+ // "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ // "docs.file=" + getReuters20LinesFile(),
+ // "work.dir=" + getWorkDir().FullName.Replace(@"\\\\", "/"), // Fix Windows path
+ // "content.source.forever=false",
+ // "directory=RAMDirectory",
+ // "AnalyzerFactory(name:'" + singleQuoteEscapedName + "', " + @params + ")",
+ // "NewAnalyzer('" + singleQuoteEscapedName + "')",
+ // "CreateIndex",
+ // "{ \"AddDocs\" AddDoc > : * "
+ //};
+ //String singleQuoteEscapedName = name.Replace("'", @"\'");
+ String singleQuoteEscapedName = name.Replace("'", @"\'");
+ String[] algLines = {
+ "content.source=Lucene.Net.Benchmarks.ByTask.Feeds.LineDocSource, Lucene.Net.Benchmark",
+ "docs.file=" + getReuters20LinesFile(),
+ "work.dir=" + getWorkDir().FullName.Replace(@"\", "/"), // Fix Windows path
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "AnalyzerFactory(name:'" + singleQuoteEscapedName + "', " + @params + ")",
+ "NewAnalyzer('" + singleQuoteEscapedName + "')",
+ "CreateIndex",
+ "{ \"AddDocs\" AddDoc > : * "
+ };
+ return algLines;
+ }
+
+ [Test]
+ public void TestAnalyzerFactory()
+ {
+ String text = "Fortieth, Quarantième, Cuadragésimo";
+ Benchmark benchmark = execBenchmark(getAnalyzerFactoryConfig
+ ("ascii folded, pattern replaced, standard tokenized, downcased, bigrammed.'analyzer'",
+ "positionIncrementGap:100,offsetGap:1111,"
+ + "MappingCharFilter(mapping:'test-mapping-ISOLatin1Accent-partial.txt'),"
+ + "PatternReplaceCharFilterFactory(pattern:'e(\\\\\\\\S*)m',replacement:\"$1xxx$1\"),"
+ + "StandardTokenizer,LowerCaseFilter,NGramTokenFilter(minGramSize:2,maxGramSize:2)"));
+ BaseTokenStreamTestCase.AssertAnalyzesTo(benchmark.RunData.Analyzer, text,
+ new String[] { "fo", "or", "rt", "ti", "ie", "et", "th",
+ "qu", "ua", "ar", "ra", "an", "nt", "ti", "ix", "xx", "xx", "xe",
+ "cu", "ua", "ad", "dr", "ra", "ag", "gs", "si", "ix", "xx", "xx", "xs", "si", "io"});
+ }
+
+ private String getReuters20LinesFile()
+ {
+ return getWorkDirResourcePath("reuters.first20.lines.txt");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksParse.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksParse.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksParse.cs
new file mode 100644
index 0000000..e604cef
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksParse.cs
@@ -0,0 +1,178 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test very simply that perf tasks are parses as expected.
+ /// </summary>
+ public class TestPerfTasksParse : LuceneTestCase
+ {
+ static readonly String NEW_LINE = Environment.NewLine;
+ static readonly String INDENT = " ";
+
+ // properties in effect in all tests here
+ static readonly String propPart =
+ INDENT + "directory=RAMDirectory" + NEW_LINE +
+ INDENT + "print.props=false" + NEW_LINE
+ ;
+
+ /** Test the repetiotion parsing for parallel tasks */
+ [Test]
+ public void TestParseParallelTaskSequenceRepetition()
+ {
+ String taskStr = "AddDoc";
+ String parsedTasks = "[ " + taskStr + " ] : 1000";
+ Benchmark benchmark = new Benchmark(new StringReader(propPart + parsedTasks));
+ Algorithm alg = benchmark.Algorithm;
+ IList<PerfTask> algTasks = alg.ExtractTasks();
+ bool foundAdd = false;
+ foreach (PerfTask task in algTasks)
+ {
+ if (task.toString().IndexOf(taskStr) >= 0)
+ {
+ foundAdd = true;
+ }
+ if (task is TaskSequence)
+ {
+ assertEquals("repetions should be 1000 for " + parsedTasks, 1000, ((TaskSequence)task).Repetitions);
+ assertTrue("sequence for " + parsedTasks + " should be parallel!", ((TaskSequence)task).IsParallel);
+ }
+ assertTrue("Task " + taskStr + " was not found in " + alg.toString(), foundAdd);
+ }
+ }
+
+ /** Test the repetiotion parsing for sequential tasks */
+ [Test]
+ public void TestParseTaskSequenceRepetition()
+ {
+ String taskStr = "AddDoc";
+ String parsedTasks = "{ " + taskStr + " } : 1000";
+ Benchmark benchmark = new Benchmark(new StringReader(propPart + parsedTasks));
+ Algorithm alg = benchmark.Algorithm;
+ IList<PerfTask> algTasks = alg.ExtractTasks();
+ bool foundAdd = false;
+ foreach (PerfTask task in algTasks)
+ {
+ if (task.toString().IndexOf(taskStr) >= 0)
+ {
+ foundAdd = true;
+ }
+ if (task is TaskSequence)
+ {
+ assertEquals("repetions should be 1000 for " + parsedTasks, 1000, ((TaskSequence)task).Repetitions);
+ assertFalse("sequence for " + parsedTasks + " should be sequential!", ((TaskSequence)task).IsParallel);
+ }
+ assertTrue("Task " + taskStr + " was not found in " + alg.toString(), foundAdd);
+ }
+ }
+
+ public class MockContentSource : ContentSource
+ {
+ public override DocData GetNextDocData(DocData docData)
+ {
+ return docData;
+ }
+
+ protected override void Dispose(bool disposing) { }
+ }
+
+ public class MockQueryMaker : AbstractQueryMaker
+ {
+ protected override Query[] PrepareQueries()
+ {
+ return new Query[0];
+ }
+ }
+
+ /// <summary>Test the parsing of example scripts</summary>
+ [Test]
+ public void TestParseExamples()
+ {
+ // LUCENENET specific
+ // Rather than relying on a file path somewhere, we store the
+ // files zipped in an embedded resource and unzip them to a
+ // known temp directory for the test.
+ DirectoryInfo examplesDir = CreateTempDir("test-parse-examples");
+ using (var stream = GetType().getResourceAsStream("conf.zip"))
+ {
+ TestUtil.Unzip(stream, examplesDir);
+ }
+
+ // hackedy-hack-hack
+ bool foundFiles = false;
+
+ foreach (FileInfo algFile in examplesDir.EnumerateFiles("*.alg"))
+ {
+ try
+ {
+ Config config = new Config(new StreamReader(new FileStream(algFile.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8));
+ String contentSource = config.Get("content.source", null);
+ if (contentSource != null)
+ {
+ if (Type.GetType(contentSource) == null)
+ throw new TypeLoadException(contentSource);
+ }
+ config.Set("work.dir", CreateTempDir(LuceneTestCase.TestClass.Name).FullName);
+ config.Set("content.source", typeof(MockContentSource).AssemblyQualifiedName);
+ String dir = config.Get("content.source", null);
+ if (dir != null)
+ {
+ if (Type.GetType(dir) == null)
+ throw new TypeLoadException(dir);
+ }
+ config.Set("directory", typeof(RAMDirectory).AssemblyQualifiedName);
+ if (config.Get("line.file.out", null) != null)
+ {
+ config.Set("line.file.out", CreateTempFile("linefile", ".txt").FullName);
+ }
+ string queryMaker = config.Get("query.maker", null);
+ if (queryMaker != null)
+ {
+ if (Type.GetType(queryMaker) == null)
+ throw new TypeLoadException(queryMaker);
+
+ config.Set("query.maker", typeof(MockQueryMaker).AssemblyQualifiedName);
+ }
+ PerfRunData data = new PerfRunData(config);
+ new Algorithm(data);
+ }
+ catch (Exception t)
+ {
+ throw new Exception("Could not parse sample file: " + algFile, t);
+ }
+ foundFiles = true;
+ }
+ if (!foundFiles)
+ {
+ fail("could not find any .alg files!");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/StreamUtilsTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/StreamUtilsTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/StreamUtilsTest.cs
new file mode 100644
index 0000000..eb2aaec
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/StreamUtilsTest.cs
@@ -0,0 +1,149 @@
+using ICSharpCode.SharpZipLib.BZip2;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class StreamUtilsTest : BenchmarkTestCase
+ {
+ private static readonly String TEXT = "Some-Text...";
+ private DirectoryInfo testDir;
+
+ [Test]
+ public void TestGetInputStreamPlainText()
+ {
+ assertReadText(rawTextFile("txt"));
+ assertReadText(rawTextFile("TXT"));
+ }
+
+ [Test]
+ public void TestGetInputStreamGzip()
+ {
+ assertReadText(rawGzipFile("gz"));
+ assertReadText(rawGzipFile("gzip"));
+ assertReadText(rawGzipFile("GZ"));
+ assertReadText(rawGzipFile("GZIP"));
+ }
+
+ [Test]
+ public void TestGetInputStreamBzip2()
+ {
+ assertReadText(rawBzip2File("bz2"));
+ assertReadText(rawBzip2File("bzip"));
+ assertReadText(rawBzip2File("BZ2"));
+ assertReadText(rawBzip2File("BZIP"));
+ }
+
+ [Test]
+ public void TestGetOutputStreamBzip2()
+ {
+ assertReadText(autoOutFile("bz2"));
+ assertReadText(autoOutFile("bzip"));
+ assertReadText(autoOutFile("BZ2"));
+ assertReadText(autoOutFile("BZIP"));
+ }
+
+ [Test]
+ public void TestGetOutputStreamGzip()
+ {
+ assertReadText(autoOutFile("gz"));
+ assertReadText(autoOutFile("gzip"));
+ assertReadText(autoOutFile("GZ"));
+ assertReadText(autoOutFile("GZIP"));
+ }
+
+ [Test]
+ public void TestGetOutputStreamPlain()
+ {
+ assertReadText(autoOutFile("txt"));
+ assertReadText(autoOutFile("text"));
+ assertReadText(autoOutFile("TXT"));
+ assertReadText(autoOutFile("TEXT"));
+ }
+
+ private FileInfo rawTextFile(String ext)
+ {
+ FileInfo f = new FileInfo(Path.Combine(testDir.FullName, "testfile." + ext));
+ using (TextWriter w = new StreamWriter(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), Encoding.UTF8))
+ w.WriteLine(TEXT);
+ return f;
+ }
+
+ private FileInfo rawGzipFile(String ext)
+ {
+ FileInfo f = new FileInfo(Path.Combine(testDir.FullName, "testfile." + ext));
+ using (Stream os = new GZipStream(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), CompressionMode.Compress)) //new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, new FileOutputStream(f));
+ writeText(os);
+ return f;
+ }
+
+ private FileInfo rawBzip2File(String ext)
+ {
+ FileInfo f = new FileInfo(Path.Combine(testDir.FullName, "testfile." + ext));
+ Stream os = new BZip2OutputStream(new FileStream(f.FullName, FileMode.Create, FileAccess.Write)); // new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.BZIP2, new FileOutputStream(f));
+ writeText(os);
+ return f;
+ }
+
+ private FileInfo autoOutFile(String ext)
+ {
+ FileInfo f = new FileInfo(Path.Combine(testDir.FullName, "testfile." + ext));
+ Stream os = StreamUtils.GetOutputStream(f);
+ writeText(os);
+ return f;
+ }
+
+ private void writeText(Stream os)
+ {
+ TextWriter w = new StreamWriter(os, Encoding.UTF8);
+ w.WriteLine(TEXT);
+ w.Dispose();
+ }
+
+ private void assertReadText(FileInfo f)
+ {
+ Stream ir = StreamUtils.GetInputStream(f);
+ TextReader r = new StreamReader(ir, Encoding.UTF8);
+ String line = r.ReadLine();
+ assertEquals("Wrong text found in " + f.Name, TEXT, line);
+ r.Dispose();
+ }
+
+ public override void SetUp()
+ {
+ base.SetUp();
+ testDir = new DirectoryInfo(Path.Combine(getWorkDir().FullName, "ContentSourceTest"));
+ TestUtil.Rm(testDir);
+ //assertTrue(testDir.mkdirs());
+ testDir.Create();
+ assertTrue(Directory.Exists(testDir.FullName));
+ }
+
+ public override void TearDown()
+ {
+ TestUtil.Rm(testDir);
+ base.TearDown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/TestConfig.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/TestConfig.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/TestConfig.cs
new file mode 100644
index 0000000..75f16a9
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Utils/TestConfig.cs
@@ -0,0 +1,37 @@
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class TestConfig : LuceneTestCase
+ {
+ [Test]
+ public void TestAbsolutePathNamesWindows()
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["work.dir1"] = "c:\\temp";
+ props["work.dir2"] = "c:/temp";
+ Config conf = new Config(props);
+ assertEquals("c:\\temp", conf.Get("work.dir1", ""));
+ assertEquals("c:/temp", conf.Get("work.dir2", ""));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/conf.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/conf.zip b/src/Lucene.Net.Tests.Benchmark/ByTask/conf.zip
new file mode 100644
index 0000000..9b5755e
Binary files /dev/null and b/src/Lucene.Net.Tests.Benchmark/ByTask/conf.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/reuters.first20.lines.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/reuters.first20.lines.txt b/src/Lucene.Net.Tests.Benchmark/ByTask/reuters.first20.lines.txt
new file mode 100644
index 0000000..41b04b3
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/reuters.first20.lines.txt
@@ -0,0 +1,20 @@
+BAHIA COCOA REVIEW 19870226200101 Showers continued throughout the week in the Bahia cocoa zone, alleviating the drought since early January and improving prospects for the coming temporao, although normal humidity levels have not been restored, Comissaria Smith said in its weekly review. The dry period means the temporao will be late this year. Arrivals for the week ended February 22 were 155,221 bags of 60 kilos making a cumulative total for the season of 5.93 mln against 5.81 at the same stage last year. Again it seems that cocoa delivered earlier on consignment was included in the arrivals figures. Comissaria Smith said there is still some doubt as to how much old crop cocoa is still available as harvesting has practically come to an end. With total Bahia crop estimates around 6.4 mln bags and sales standing at almost 6.2 mln there are a few hundred thousand bags still in the hands of farmers, middlemen, exporters and processors. There are doubts as to how much o
f this cocoa would be fit for export as shippers are now experiencing dificulties in obtaining +Bahia superior+ certificates. In view of the lower quality over recent weeks farmers have sold a good part of their cocoa held on consignment. Comissaria Smith said spot bean prices rose to 340 to 350 cruzados per arroba of 15 kilos. Bean shippers were reluctant to offer nearby shipment and only limited sales were booked for March shipment at 1,750 to 1,780 dlrs per tonne to ports to be named. New crop sales were also light and all to open ports with June/July going at 1,850 and 1,880 dlrs and at 35 and 45 dlrs under New York july, Aug/Sept at 1,870, 1,875 and 1,880 dlrs per tonne FOB. Routine sales of butter were made. March/April sold at 4,340, 4,345 and 4,350 dlrs. April/May butter went at 2.27 times New York May, June/July at 4,400 and 4,415 dlrs, Aug/Sept at 4,351 to 4,450 dlrs and at 2.27 and 2.28 times New York Sept and Oct/Dec at 4,480 dlrs and 2.27 times N
ew York Dec, Comissaria Smith said. Destinations were the U.S., Covertible currency areas, Uruguay and open ports. Cake sales were registered at 785 to 995 dlrs for March/April, 785 dlrs for May, 753 dlrs for Aug and 0.39 times New York Dec for Oct/Dec. Buyers were the U.S., Argentina, Uruguay and convertible currency areas. Liquor sales were limited with March/April selling at 2,325 and 2,380 dlrs, June/July at 2,375 dlrs and at 1.25 times New York July, Aug/Sept at 2,400 dlrs and at 1.25 times New York Sept and Oct/Dec at 1.25 times New York Dec, Comissaria Smith said. Total Bahia sales are currently estimated at 6.13 mln bags against the 1986/87 crop and 1.06 mln bags against the 1987/88 crop. Final figures for the period to February 28 are expected to be published by the Brazilian Cocoa Trade Commission after carnival which ends midday on February 27. Reuter 
+STANDARD OIL <SRD> TO FORM FINANCIAL UNIT 19870226200220 Standard Oil Co and BP North America Inc said they plan to form a venture to manage the money market borrowing and investment activities of both companies. BP North America is a subsidiary of British Petroleum Co Plc <BP>, which also owns a 55 pct interest in Standard Oil. The venture will be called BP/Standard Financial Trading and will be operated by Standard Oil under the oversight of a joint management committee. Reuter 
+COBANCO INC <CBCO> YEAR NET 19870226201859 Shr 34 cts vs 1.19 dlrs Net 807,000 vs 2,858,000 Assets 510.2 mln vs 479.7 mln Deposits 472.3 mln vs 440.3 mln Loans 299.2 mln vs 327.2 mln Note: 4th qtr not available. Year includes 1985 extraordinary gain from tax carry forward of 132,000 dlrs, or five cts per shr. Reuter 
+WORLD MARKET PRICE FOR UPLAND COTTON - USDA 19870226213846 The U.S. Agriculture Department announced the prevailing world market price, adjusted to U.S. quality and location, for Strict Low Middling, 1-1/16 inch upland cotton at 52.69 cts per lb, to be in effect through midnight March 5. The adjusted world price is at average U.S. producing locations (near Lubbock, Texas) and will be further adjusted for other qualities and locations. The price will be used in determining First Handler Cotton Certificate payment rates. Based on data for the week ended February 26, the adjusted world price for upland cotton is determined as follows, in cts per lb -- Northern European Price 66.32 Adjustments -- Average U.S. spot mkt location 10.42 SLM 1-1/16 inch cotton 1.80 Average U.S. location 0.53 Sum of adjustments 12.75 Adjusted world price 53.57 Reuter 
+SUGAR QUOTA IMPORTS DETAILED -- USDA 19870226213854 The U.S. Agriculture Department said cumulative sugar imports from individual countries during the 1987 quota year, which began January 1, 1987 and ends December 31, 1987 were as follows, with quota allocations for the quota year in short tons, raw value -- CUMULATIVE QUOTA 1987 IMPORTS ALLOCATIONS ARGENTINA nil 39,130 AUSTRALIA nil 75,530 BARBADOS nil 7,500 BELIZE nil 10,010 BOLIVIA nil 7,500 BRAZIL nil 131,950 CANADA nil 18,876 QUOTA 1987 IMPORTS ALLOCATIONS COLOMBIA 103 21,840 CONGO nil 7,599 COSTA RICA nil 17,583 IVORY COAST nil 7,500 DOM REP 5,848 160,160 ECUADOR nil 10,010 EL SALVADOR nil 26,019.8
FIJI nil 25,190 GABON nil 7,500 QUOTA 1987 IMPORTS ALLOCATIONS GUATEMALA nil 43,680 GUYANA nil 10,920 HAITI nil 7,500 HONDURAS nil 15,917.2 INDIA nil 7,500 JAMAICA nil 10,010 MADAGASCAR nil 7,500 MALAWI nil 9,,100 QUOTA 1987 IMPORTS ALLOCATIONS MAURITIUS nil 10,920 MEXICO 37 7,500 MOZAMBIQUE nil 11,830 PANAMA nil 26,390 PAPUA NEW GUINEA nil 7,500 PARAGUAY nil 7,500 PERU nil 37,310 PHILIPPINES nil 143,780 ST.CHRISTOPHER- NEVIS nil 7,500 QUOTA 1987 IMPORTS ALLOCATIONS SWAZILAND nil
14,560 TAIWAN nil 10,920 THAILAND nil 12,740 TRINIDAD-TOBAGO nil 7,500 URUGUAY nil 7,500 ZIMBABWE nil 10,920 Reuter 
+GRAIN SHIPS LOADING AT PORTLAND 19870226213903 There were seven grain ships loading and six ships were waiting to load at Portland, according to the Portland Merchants Exchange. Reuter 
+IRAN ANNOUNCES END OF MAJOR OFFENSIVE IN GULF WAR 19870226214000 Iran announced tonight that its major offensive against Iraq in the Gulf war had ended after dealing savage blows against the Baghdad government. The Iranian news agency IRNA, in a report received in London, said the operation code-named Karbala-5 launched into Iraq on January 9 was now over. It quoted a joint statewment by the Iranian Army and Revolutionary Guards Corps as saying that their forces had "dealt one of the severest blows on the Iraqi war machine in the history of the Iraq-imposed war." The statement by the Iranian High Command appeared to herald the close of an assault on the port city of Basra in southern Iraq. "The operation was launched at a time when the Baghdad government was spreading extensive propaganda on the resistance power of its army...," said the statement quoted by IRNA. It claimed massive victories in the seven-week offensive and called on supporters of Baghdad to "come
to their senses" and discontinue support for what it called the tottering regime in Iraq. Iran said its forces had "liberated" 155 square kilometers of enemy-occupied territory during the 1987 offensive and taken over islands, townships, rivers and part of a road leading into Basra. The Iranian forces "are in full control of these areas," the statement said. It said 81 Iraqi brigades and battalions were totally destroyed, along with 700 tanks and 1,500 other vehicles. The victory list also included 80 warplanes downed, 250 anti- aircraft guns and 400 pieces of military hardware destroyed and the seizure of 220 tanks and armoured personnel carriers. Reuter 
+MERIDIAN BANCORP INC <MRDN> SETS REGULAR PAYOUT 19870226214034 Qtly div 25 cts vs 25 cts prior Pay April one Record March 15 Reuter 
+U.S. BANK DISCOUNT BORROWINGS 310 MLN DLRS 19870226214134 U.S. bank discount window borrowings less extended credits averaged 310 mln dlrs in the week to Wednesday February 25, the Federal Reserve said. The Fed said that overall borrowings in the week fell 131 mln dlrs to 614 mln dlrs, with extended credits up 10 mln dlrs at 304 mln dlrs. The week was the second half of a two-week statement period. Net borrowings in the prior week averaged 451 mln dlrs. Commenting on the two-week statement period ended February 25, the Fed said that banks had average net free reserves of 644 mln dlrs a day, down from 1.34 billion two weeks earlier. A Federal Reserve spokesman told a press briefing that there were no large single day net misses in the Fed's reserve projections in the week to Wednesday. He said that natural float had been "acting a bit strangely" for this time of year, noting that there had been poor weather during the latest week. The spokesman said that natural f
loat ranged from under 500 mln dlrs on Friday, for which he could give no reason, to nearly one billion dlrs on both Thursday and Wednesday. The Fed spokeman could give no reason for Thursday's high float, but he said that about 750 mln dlrs of Wednesday's float figure was due to holdover and transportation float at two widely separated Fed districts. For the week as a whole, he said that float related as of adjustments were "small," adding that they fell to a negative 750 mln dlrs on Tuesday due to a number of corrections for unrelated cash letter errors in six districts around the country. The spokesman said that on both Tuesday and Wednesday, two different clearing banks had system problems and the securities and Federal funds wires had to be held open until about 2000 or 2100 EST on both days. However, he said that both problems were cleared up during both afternoons and there was no evidence of any reserve impact. During the week ended Wednesday, 45 pct of n
et discount window borrowings were made by the smallest banks, with 30 pct by the 14 large money center banks and 25 pct by large regional institutions. On Wednesday, 55 pct of the borrowing was accounted for by the money center banks, with 30 pct by the large regionals and 15 pct by the smallest banks. The Fed spokesman said the banking system had excess reserves on Thursday, Monday and Tuesday and a deficit on Friday and Wedndsday. That produced a small daily average deficit for the week as a whole. For the two-week period, he said there were relatively high excess reserves on a daily avearge, almost all of which were at the smallest banks. Reuter 
+AMERICAN EXPRESS <AXP> SEEN IN POSSIBLE SPINNOFF 19870226214313 American Express Co remained silent on market rumors it would spinoff all or part of its Shearson Lehman Brothers Inc, but some analysts said the company may be considering such a move because it is unhappy with the market value of its stock. American Express stock got a lift from the rumor, as the market calculated a partially public Shearson may command a good market value, thereby boosting the total value of American Express. The rumor also was accompanied by talk the financial services firm would split its stock and boost its dividend. American Express closed on the New York Stock Exchange at 72-5/8, up 4-1/8 on heavy volume. American Express would not comment on the rumors or its stock activity. Analysts said comments by the company at an analysts' meeting Tuesday helped fuel the rumors as did an announcement yesterday of management changes. At the meeting, company officials said American Expres
s stock is undervalued and does not fully reflect the performance of Shearson, according to analysts. Yesterday, Shearson said it was elevating its chief operating officer, Jeffery Lane, to the added position of president, which had been vacant. It also created four new positions for chairmen of its operating divisions. Analysts speculated a partial spinoff would make most sense, contrary to one variation on market rumors of a total spinoff. Some analysts, however, disagreed that any spinoff of Shearson would be good since it is a strong profit center for American Express, contributing about 20 pct of earnings last year. "I think it is highly unlikely that American Express is going to sell shearson," said Perrin Long of Lipper Analytical. He questioned what would be a better investment than "a very profitable securities firm." Several analysts said American Express is not in need of cash, which might be the only reason to sell a part of a strong asset. But ot
hers believe the company could very well of considered the option of spinning out part of Shearson, and one rumor suggests selling about 20 pct of it in the market. Larry Eckenfelder of Prudential-Bache Securities said he believes American Express could have considered a partial spinoff in the past. "Shearson being as profitable as it is would have fetched a big premium in the market place. Shearson's book value is in the 1.4 mln dlr range. Shearson in the market place would probably be worth three to 3.5 bilion dlrs in terms of market capitalization," said Eckenfelder. Some analysts said American Express could use capital since it plans to expand globally. "They have enormous internal growth plans that takes capital. You want your stock to reflect realistic valuations to enhance your ability to make all kinds of endeavors down the road," said E.F. Hutton Group analyst Michael Lewis. "They've outlined the fact that they're investing heavily in the future, which g
oes heavily into the international arena," said Lewis. "...That does not preclude acquisitions and divestitures along the way," he said. Lewis said if American Express reduced its exposure to the brokerage business by selling part of shearson, its stock might better reflect other assets, such as the travel related services business. "It could find its true water mark with a lesser exposure to brokerage. The value of the other components could command a higher multiple because they constitute a higher percentage of the total operating earnings of the company," he said. Lewis said Shearson contributed 316 mln in after-tax operating earnings, up from about 200 mln dlrs in 1985. Reuter 
+OHIO MATTRESS <OMT> MAY HAVE LOWER 1ST QTR NET 19870226201915 Ohio Mattress Co said its first quarter, ending February 28, profits may be below the 2.4 mln dlrs, or 15 cts a share, earned in the first quarter of fiscal 1986. The company said any decline would be due to expenses related to the acquisitions in the middle of the current quarter of seven licensees of Sealy Inc, as well as 82 pct of the outstanding capital stock of Sealy. Because of these acquisitions, it said, first quarter sales will be substantially higher than last year's 67.1 mln dlrs. Noting that it typically reports first quarter results in late march, said the report is likely to be issued in early April this year. It said the delay is due to administrative considerations, including conducting appraisals, in connection with the acquisitions. Reuter 
+U.S. M-1 MONEY SUPPLY ROSE 2.1 BILLION DLRS 19870226214435 U.S. M-1 money supply rose 2.1 billion dlrs to a seasonally adjusted 736.7 billion dlrs in the February 16 week, the Federal Reserve said. The previous week's M-1 level was revised to 734.6 billion dlrs from 734.2 billion dlrs, while the four-week moving average of M-1 rose to 735.0 billion dlrs from 733.5 billion. Economists polled by Reuters said that M-1 should be anywhere from down four billion dlrs to up 2.3 billion dlrs. The average forecast called for a 300 mln dlr M-1 rise. Reuter 
+GENERAL BINDING <GBND> IN MARKETING AGREEMENT 19870226214508 General Binding Corp said it reached a marketing agreement with Varitronic Systems Inc, a manufacturer and marketer of electronic lettering systems. Under terms of the agreement, General Binding will carry Varitronics' Merlin Express Presentation Lettering System, a portable, battery-operated lettering system which produces type on adhesive-backed tape. Reuter 
+LIBERTY ALL-STAR <USA> SETS INITIAL PAYOUT 19870226214544 Liberty All-Star Equity Fund said it declared an initial dividend of five cts per share, payable April two to shareholders of record March 20. It said the dividend includes a quarterly dividend of three cts a share and a special payout of two cts a share, which covers the period from November three, 1986, when the fund began operations, to December 31, 1986. The fund said its quarterly dividend rate may fluctuate in the future. Reuter 
+COCA COLA <KO> UNIT AND WORLD FILM IN VENTURE 19870226214745 Coca-Cola Co's Entertainment Business Sector Inc unit said it formed a joint venture with an affiliate of World Film Services to acquire, produce and distribute television programming around the world. World Film Services was formed by chairman John Heyman in 1963 to produce films. Reuter 
+FORD MOTOR CREDIT <F> TO REDEEM DEBENTURES 19870226214753 Ford Motor Co said its Ford Motor Credit Co on April One will redeem 4.0 mln dlrs of its 8.70 pct debentures due April 1, 1999. It said the debentures are redeemable at a price of 100 pct of the principal. Because April 1, 1987 is an interest payment date on the debentures, no accrued interest will be payable on the redemption date as part of the redemption proceeds. Debentures will be selected for redemption on a pro rata basis, Ford said. Reuter 
+STERLING SOFTWARE <SSW> NOTE HOLDERS OK BUY 19870226214802 Sterling Software Inc said it received consent of a majority of the holders of its eight pct convertible sernior subordinated debentures required to purchase shares of its common. The company said it may now buy its stock at its discretion depending on market conditions. Reuter 
+<SCHULT HOMES CORP> MAKES INITIAL STOCK OFFER 19870226214818 Schult Homes Corp announced an initial public offering of 833,334 units at five dlrs per unit, said Janney Montgomery Scott Inc and Woolcott and Co, managing underwriters of the offering. They said each unit consists of one common share and one warrant to buy one-half share of common. The warrant will entitle holders to buy one-half common share at 5.50 dlrs per full share from March one, 1988, to September one, 1989, and thereafter at 6.50 dlrs per full share until March 1991, they said. Reuter 
+FLUOR <FLR> UNIT GETS CONSTRUCTION CONTRACT 19870226214826 Fluor Corp said its Fluor Daniel unit received a contract from Union Carbide Corp <UK> covering design, procurement and construction of a 108 megawatt combined cycle cogeneration facility in Seadrift, Texas. The value of the contract was not disclosed. Reuter 
+SUFFIELD FINANCIAL CORP <SFCP> SELLS STOCK 19870226214835 Suffield Financial Corp said Jon Googel and Benjamin Sisti of Colonial Realty, West Hartford, Conn., purchased 175,900 shares of its stock for 3,416,624. The company said the purchase equals 5.2 pct of its outstanding shares. Reuter 
[14/33] lucenenet git commit: Lucene.Net.Benchmark: Created a simple
English number formatter to spell out numbers into words. Since we don't need
localization,
this is a sufficient replacement for the ICU RuleBasedNumberFormatter.
Posted by ni...@apache.org.
Lucene.Net.Benchmark: Created a simple English number formatter to spell out numbers into words. Since we don't need localization, this is a sufficient replacement for the ICU RuleBasedNumberFormatter.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/1cfbd8b7
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/1cfbd8b7
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/1cfbd8b7
Branch: refs/heads/master
Commit: 1cfbd8b7c35c7f1ae2bd616b44d752eef4d7d180
Parents: a60c5ef
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Tue Aug 1 21:11:54 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Wed Aug 2 09:55:15 2017 +0700
----------------------------------------------------------------------
.../ByTask/Feeds/LongToEnglishContentSource.cs | 6 +-
.../ByTask/Feeds/LongToEnglishQueryMaker.cs | 4 +-
.../Lucene.Net.Benchmark.csproj | 1 +
.../Support/EnglishNumberFormatExtensions.cs | 186 +++++++++++++++++++
.../Lucene.Net.Tests.Benchmark.csproj | 1 +
.../TestEnglishNumberFormatExtensions.cs | 38 ++++
6 files changed, 231 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/1cfbd8b7/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
index fadab82..7c407a2 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
@@ -1,4 +1,5 @@
-using System;
+using Lucene.Net.Support;
+using System;
using System.Globalization;
namespace Lucene.Net.Benchmarks.ByTask.Feeds
@@ -55,8 +56,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
}
}
- // LUCENENET TODO: Rules based number formatting...(from ICU)
- docData.Body = curCounter.ToString(); //rnbf.format(curCounter);
+ docData.Body = curCounter.ToWords(); //rnbf.format(curCounter);
docData.Name = "doc_" + curCounter.ToString(CultureInfo.InvariantCulture);
docData.Title = "title_" + curCounter.ToString(CultureInfo.InvariantCulture);
docData.SetDate(new DateTime());
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/1cfbd8b7/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
index f565eb8..78ac924 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
@@ -4,6 +4,7 @@ using Lucene.Net.Benchmarks.ByTask.Tasks;
using Lucene.Net.Benchmarks.ByTask.Utils;
using Lucene.Net.QueryParsers.Classic;
using Lucene.Net.Search;
+using Lucene.Net.Support;
using Lucene.Net.Util;
using System;
@@ -48,9 +49,8 @@ namespace Lucene.Net.Benchmarks.ByTask.Feeds
{
lock (this)
{
- // LUCENENET TODO: Rules based number formatter (from ICU)
//return parser.Parse("" + rnbf.format(GetNextCounter()) + "");
- return m_parser.Parse(GetNextCounter().ToString());
+ return m_parser.Parse(GetNextCounter().ToWords());
}
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/1cfbd8b7/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
index 0241099..f00cd18 100644
--- a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
@@ -159,6 +159,7 @@
<Compile Include="Quality\Utils\QualityQueriesFinder.cs" />
<Compile Include="Quality\Utils\SimpleQQParser.cs" />
<Compile Include="Quality\Utils\SubmissionReport.cs" />
+ <Compile Include="Support\EnglishNumberFormatExtensions.cs" />
<Compile Include="Utils\ExtractReuters.cs" />
<Compile Include="Utils\ExtractWikipedia.cs" />
<Compile Include="..\CommonAssemblyInfo.cs">
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/1cfbd8b7/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs b/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
new file mode 100644
index 0000000..71362f0
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/EnglishNumberFormatExtensions.cs
@@ -0,0 +1,186 @@
+using System;
+using System.Text;
+
+namespace Lucene.Net.Support
+{
+ /// <summary>
+ /// Extension methods to spell out numbers into English.
+ /// <para/>
+ /// Inspiration: https://stackoverflow.com/a/2601001
+ /// </summary>
+ public static class EnglishNumberFormatExtensions
+ {
+ private const long Quadrillion = Trillion * 1000;
+ private const long Trillion = Billion * 1000;
+ private const long Billion = Million * 1000;
+ private const long Million = Thousand * 1000;
+ private const long Thousand = Hundred * 10;
+ private const long Hundred = 100;
+
+ /// <summary>
+ /// Returns the spelled-out English words for the provided <paramref name="value"/>.
+ /// </summary>
+ public static string ToWords(this int value)
+ {
+ return ToWords((long)value);
+ }
+
+ /// <summary>
+ /// Returns the spelled-out English words for the provided <paramref name="value"/>.
+ /// </summary>
+ public static string ToWords(this long value)
+ {
+ return ToWords(value, new StringBuilder()).ToString();
+ }
+ private static StringBuilder ToWords(long value, StringBuilder builder)
+ {
+ if (value == 0) builder.Append("zero");
+
+ if (value < 0)
+ {
+ builder.Append("negative ");
+ ToWords(Math.Abs(value), builder);
+ }
+
+ long unit = 0;
+
+ if (value >= Quadrillion)
+ {
+ unit = (value / Quadrillion);
+ value -= unit * Quadrillion;
+
+ ToWords(unit, builder);
+ builder.Append(" quadrillion");
+ if (value > 0) builder.Append(" ");
+ }
+
+ if (value >= Trillion)
+ {
+ unit = (value / Trillion);
+ value -= unit * Trillion;
+
+ ToWords(unit, builder);
+ builder.Append(" trillion");
+ if (value > 0) builder.Append(" ");
+ }
+
+ if (value >= Billion)
+ {
+ unit = (value / Billion);
+ value -= unit * Billion;
+
+ ToWords(unit, builder);
+ builder.Append(" billion");
+ if (value > 0) builder.Append(" ");
+ }
+
+ if (value >= Million)
+ {
+ unit = (value / Million);
+ value -= unit * Million;
+
+ ToWords(unit, builder);
+ builder.Append(" million");
+ if (value > 0) builder.Append(" ");
+ }
+
+ if (value >= Thousand)
+ {
+ unit = (value / Thousand);
+ value -= unit * Thousand;
+
+ ToWords(unit, builder);
+ builder.Append(" thousand");
+ if (value > 0) builder.Append(" ");
+ }
+
+ if (value >= Hundred)
+ {
+ unit = (value / Hundred);
+ value -= unit * Hundred;
+
+ ToWords(unit, builder);
+ builder.Append(" hundred");
+ if (value > 0) builder.Append(" ");
+ }
+
+ if (value >= 90)
+ {
+ value -= 90;
+ builder.Append("ninety");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 80)
+ {
+ value -= 80;
+ builder.Append("eighty");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 70)
+ {
+ value -= 70;
+ builder.Append("seventy");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 60)
+ {
+ value -= 60;
+ builder.Append("sixty");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 50)
+ {
+ value -= 50;
+ builder.Append("fifty");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 40)
+ {
+ value -= 40;
+ builder.Append("forty");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 30)
+ {
+ value -= 30;
+ builder.Append("thirty");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value >= 20)
+ {
+ value -= 20;
+ builder.Append("twenty");
+ if (value > 0) builder.Append("-");
+ }
+
+ if (value == 19) builder.Append("nineteen");
+ if (value == 18) builder.Append("eighteen");
+ if (value == 17) builder.Append("seventeen");
+ if (value == 16) builder.Append("sixteen");
+ if (value == 15) builder.Append("fifteen");
+ if (value == 14) builder.Append("fourteen");
+ if (value == 13) builder.Append("thirteen");
+ if (value == 12) builder.Append("twelve");
+ if (value == 11) builder.Append("eleven");
+ if (value == 10) builder.Append("ten");
+ if (value == 9) builder.Append("nine");
+ if (value == 8) builder.Append("eight");
+ if (value == 7) builder.Append("seven");
+ if (value == 6) builder.Append("six");
+ if (value == 5) builder.Append("five");
+ if (value == 4) builder.Append("four");
+ if (value == 3) builder.Append("three");
+ if (value == 2) builder.Append("two");
+ if (value == 1) builder.Append("one");
+
+ return builder;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/1cfbd8b7/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
index c57a59f..5c9ffe1 100644
--- a/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
+++ b/src/Lucene.Net.Tests.Benchmark/Lucene.Net.Tests.Benchmark.csproj
@@ -68,6 +68,7 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Quality\TestQualityRun.cs" />
<Compile Include="Support\TestApiConsistency.cs" />
+ <Compile Include="Support\TestEnglishNumberFormatExtensions.cs" />
<Compile Include="Support\TestExceptionSerialization.cs" />
</ItemGroup>
<ItemGroup>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/1cfbd8b7/src/Lucene.Net.Tests.Benchmark/Support/TestEnglishNumberFormatExtensions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/Support/TestEnglishNumberFormatExtensions.cs b/src/Lucene.Net.Tests.Benchmark/Support/TestEnglishNumberFormatExtensions.cs
new file mode 100644
index 0000000..68cc70a
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/Support/TestEnglishNumberFormatExtensions.cs
@@ -0,0 +1,38 @@
+using Lucene.Net.Attributes;
+using NUnit.Framework;
+
+namespace Lucene.Net.Support
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ [TestFixture]
+ public class TestEnglishNumberFormatExtensions
+ {
+ [Test, LuceneNetSpecific]
+ public void TestToWords()
+ {
+ Assert.AreEqual("twenty-one", 21.ToWords());
+ Assert.AreEqual("one thousand two hundred thirty-four", 1234.ToWords());
+ Assert.AreEqual("six million four hundred ninety-one thousand three hundred forty-eight", 6491348.ToWords());
+ Assert.AreEqual("one hundred thirty", 130.ToWords());
+ Assert.AreEqual("one hundred thirty-seven", 137.ToWords());
+ Assert.AreEqual("seven hundred forty-nine million one hundred thirty-two thousand one hundred forty-six", 749132146.ToWords());
+ Assert.AreEqual("nine hundred ninety-nine billion seven hundred forty-nine million one hundred thirty-two thousand one hundred forty-six", 999749132146.ToWords());
+ }
+ }
+}
[26/33] lucenenet git commit: BUG:
Lucene.Net.Support.TestDictionaryExtensions: Added missing embedded resource
file hyts_PropertiesTest.properties to project.json
Posted by ni...@apache.org.
BUG: Lucene.Net.Support.TestDictionaryExtensions: Added missing embedded resource file hyts_PropertiesTest.properties to project.json
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/e92525f3
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/e92525f3
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/e92525f3
Branch: refs/heads/master
Commit: e92525f35943e24233d7efdaca3e808f0552700f
Parents: 4cb35e9
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Aug 4 15:12:21 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 19:33:33 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Tests/project.json | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e92525f3/src/Lucene.Net.Tests/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/project.json b/src/Lucene.Net.Tests/project.json
index f1ff4ed..92c09c6 100644
--- a/src/Lucene.Net.Tests/project.json
+++ b/src/Lucene.Net.Tests/project.json
@@ -51,7 +51,8 @@
"Index/unsupported.29.cfs.zip",
"Index/unsupported.29.nocfs.zip",
"Store/LUCENENET521.zip",
- "Support/IO/ReadFully.txt"
+ "Support/IO/ReadFully.txt",
+ "Support/hyts_PropertiesTest.properties"
]
}
},
[31/33] lucenenet git commit: Merge branch 'master' into benchmark
Posted by ni...@apache.org.
Merge branch 'master' into benchmark
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/564a2923
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/564a2923
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/564a2923
Branch: refs/heads/master
Commit: 564a29238e5bea1794502067992a6f488785db84
Parents: 0bc62b5 14c1760
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Sun Aug 6 14:04:16 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Sun Aug 6 14:04:16 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Spatial/project.json | 4 +---
src/Lucene.Net.Tests.Spatial/project.json | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[20/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.Generated.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.Generated.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.Generated.cs
new file mode 100644
index 0000000..6f9c539
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.Generated.cs
@@ -0,0 +1,2910 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+// This code was generated by a tool.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+namespace TagSoup
+{
+ /// <summary>
+ /// This class provides a Schema that has been preinitialized with HTML
+ /// elements, attributes, and character entity declarations. All the declarations
+ /// normally provided with HTML 4.01 are given, plus some that are IE-specific
+ /// and NS4-specific. Attribute declarations of type CDATA with no default
+ /// value are not included.
+ /// </summary>
+ public class HTMLSchema : Schema
+ {
+ // HTMLModels begin
+ public const int M_AREA = 1 << 1;
+ public const int M_BLOCK = 1 << 2;
+ public const int M_BLOCKINLINE = 1 << 3;
+ public const int M_BODY = 1 << 4;
+ public const int M_CELL = 1 << 5;
+ public const int M_COL = 1 << 6;
+ public const int M_DEF = 1 << 7;
+ public const int M_FORM = 1 << 8;
+ public const int M_FRAME = 1 << 9;
+ public const int M_HEAD = 1 << 10;
+ public const int M_HTML = 1 << 11;
+ public const int M_INLINE = 1 << 12;
+ public const int M_LEGEND = 1 << 13;
+ public const int M_LI = 1 << 14;
+ public const int M_NOLINK = 1 << 15;
+ public const int M_OPTION = 1 << 16;
+ public const int M_OPTIONS = 1 << 17;
+ public const int M_P = 1 << 18;
+ public const int M_PARAM = 1 << 19;
+ public const int M_TABLE = 1 << 20;
+ public const int M_TABULAR = 1 << 21;
+ public const int M_TR = 1 << 22;
+ // HTMLModels end
+
+ /// <summary>
+ /// Returns a newly constructed HTMLSchema object independent of
+ /// any existing ones.
+ /// </summary>
+ public HTMLSchema()
+ {
+ Uri = "http://www.w3.org/1999/xhtml";
+ Prefix = "html";
+ ElementType("<pcdata>", M_EMPTY, M_PCDATA, 0);
+ ElementType("<root>", M_ROOT, M_EMPTY, 0);
+ ElementType("a", M_PCDATA|M_NOLINK, M_INLINE, 0);
+ ElementType("abbr", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("acronym", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("address", M_PCDATA|M_INLINE|M_P, M_BLOCK, 0);
+ ElementType("applet", M_PCDATA|M_PARAM|M_INLINE|M_BLOCK, M_INLINE|M_NOLINK, 0);
+ ElementType("area", M_EMPTY, M_AREA, 0);
+ ElementType("b", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("base", M_EMPTY, M_HEAD, 0);
+ ElementType("basefont", M_EMPTY, M_INLINE|M_NOLINK, 0);
+ ElementType("bdo", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("bgsound", M_EMPTY, M_HEAD, 0);
+ ElementType("big", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("blink", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("blockquote", M_PCDATA|M_INLINE|M_BLOCK, M_BLOCK, 0);
+ ElementType("body", M_PCDATA|M_INLINE|M_BLOCK, M_HTML|M_BODY, 0);
+ ElementType("br", M_EMPTY, M_INLINE|M_NOLINK, 0);
+ ElementType("button", M_PCDATA|M_INLINE|M_BLOCK, M_INLINE|M_NOLINK, 0);
+ ElementType("canvas", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("caption", M_PCDATA|M_INLINE, M_TABULAR, 0);
+ ElementType("center", M_PCDATA|M_INLINE|M_BLOCK, M_BLOCK, 0);
+ ElementType("cite", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("code", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("col", M_EMPTY, M_COL|M_TABULAR, 0);
+ ElementType("colgroup", M_COL, M_TABULAR, 0);
+ ElementType("comment", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("dd", M_PCDATA|M_INLINE|M_BLOCK, M_DEF, 0);
+ ElementType("del", M_PCDATA|M_INLINE|M_BLOCK, M_INLINE|M_BLOCKINLINE|M_BLOCK, F_RESTART);
+ ElementType("dfn", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("dir", M_LI, M_BLOCK, 0);
+ ElementType("div", M_PCDATA|M_INLINE|M_BLOCK, M_BLOCK, 0);
+ ElementType("dl", M_DEF, M_BLOCK, 0);
+ ElementType("dt", M_PCDATA|M_INLINE, M_DEF, 0);
+ ElementType("em", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("fieldset", M_PCDATA|M_LEGEND|M_INLINE|M_BLOCK, M_BLOCK, 0);
+ ElementType("font", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("form", M_PCDATA|M_INLINE|M_NOLINK|M_BLOCK|M_TR|M_CELL, M_BLOCK|M_FORM, F_NOFORCE);
+ ElementType("frame", M_EMPTY, M_FRAME, 0);
+ ElementType("frameset", M_FRAME, M_FRAME|M_HTML, 0);
+ ElementType("h1", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("h2", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("h3", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("h4", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("h5", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("h6", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("head", M_HEAD, M_HTML, 0);
+ ElementType("hr", M_EMPTY, M_BLOCK, 0);
+ ElementType("html", M_HTML, M_ROOT, 0);
+ ElementType("i", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("iframe", M_PCDATA|M_INLINE|M_BLOCK, M_INLINE|M_NOLINK, 0);
+ ElementType("img", M_EMPTY, M_INLINE|M_NOLINK, 0);
+ ElementType("input", M_EMPTY, M_INLINE|M_NOLINK, 0);
+ ElementType("ins", M_PCDATA|M_INLINE|M_BLOCK, M_INLINE|M_BLOCK, F_RESTART);
+ ElementType("isindex", M_EMPTY, M_HEAD, 0);
+ ElementType("kbd", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("label", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("legend", M_PCDATA|M_INLINE, M_LEGEND, 0);
+ ElementType("li", M_PCDATA|M_INLINE|M_BLOCK, M_LI, 0);
+ ElementType("link", M_EMPTY, M_HEAD|M_INLINE, 0);
+ ElementType("listing", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("map", M_BLOCK|M_AREA, M_INLINE, 0);
+ ElementType("marquee", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("menu", M_LI, M_BLOCK, 0);
+ ElementType("meta", M_EMPTY, M_HEAD, 0);
+ ElementType("nobr", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("noframes", M_BODY|M_BLOCK|M_INLINE, M_BLOCK|M_HTML|M_FRAME, 0);
+ ElementType("noscript", M_PCDATA|M_INLINE|M_BLOCK, M_BLOCK, 0);
+ ElementType("object", M_PCDATA|M_PARAM|M_INLINE|M_BLOCK, M_HEAD|M_INLINE|M_NOLINK, 0);
+ ElementType("ol", M_LI, M_BLOCK, 0);
+ ElementType("optgroup", M_OPTIONS, M_OPTIONS, 0);
+ ElementType("option", M_PCDATA, M_OPTION|M_OPTIONS, 0);
+ ElementType("p", M_PCDATA|M_INLINE|M_TABLE, M_BLOCK|M_P, 0);
+ ElementType("param", M_EMPTY, M_PARAM, 0);
+ ElementType("pre", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ ElementType("q", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("rb", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("rbc", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("rp", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("rt", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("rtc", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("ruby", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("s", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("samp", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("script", M_PCDATA, (int)(M_ANY & ~M_ROOT), F_CDATA);
+ ElementType("select", M_OPTIONS, M_INLINE, 0);
+ ElementType("small", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("span", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("strike", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("strong", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("style", M_PCDATA, M_HEAD|M_INLINE, F_CDATA);
+ ElementType("sub", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("sup", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("table", M_FORM|M_TABULAR, M_BLOCK|M_TABLE, F_NOFORCE);
+ ElementType("tbody", M_TR, M_TABULAR, 0);
+ ElementType("td", M_PCDATA|M_INLINE|M_BLOCK, M_CELL, 0);
+ ElementType("textarea", M_PCDATA, M_INLINE, 0);
+ ElementType("tfoot", M_TR|M_FORM|M_CELL, M_TABULAR, 0);
+ ElementType("th", M_PCDATA|M_INLINE|M_BLOCK, M_CELL, 0);
+ ElementType("thead", M_TR|M_FORM|M_CELL, M_TABULAR, 0);
+ ElementType("title", M_PCDATA, M_HEAD, 0);
+ ElementType("tr", M_FORM|M_CELL, M_TR|M_TABULAR, 0);
+ ElementType("tt", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("u", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, F_RESTART);
+ ElementType("ul", M_LI, M_BLOCK, 0);
+ ElementType("var", M_PCDATA|M_INLINE, M_INLINE|M_NOLINK, 0);
+ ElementType("wbr", M_EMPTY, M_INLINE|M_NOLINK, 0);
+ ElementType("xmp", M_PCDATA|M_INLINE, M_BLOCK, 0);
+ Parent("<pcdata>", "body");
+ Parent("html", "<root>");
+ Parent("a", "body");
+ Parent("abbr", "body");
+ Parent("acronym", "body");
+ Parent("address", "body");
+ Parent("applet", "body");
+ Parent("area", "map");
+ Parent("b", "body");
+ Parent("base", "head");
+ Parent("basefont", "body");
+ Parent("bdo", "body");
+ Parent("bgsound", "head");
+ Parent("big", "body");
+ Parent("blink", "body");
+ Parent("blockquote", "body");
+ Parent("body", "html");
+ Parent("br", "body");
+ Parent("button", "form");
+ Parent("canvas", "body");
+ Parent("caption", "table");
+ Parent("center", "body");
+ Parent("cite", "body");
+ Parent("code", "body");
+ Parent("col", "table");
+ Parent("colgroup", "table");
+ Parent("comment", "body");
+ Parent("dd", "dl");
+ Parent("del", "body");
+ Parent("dfn", "body");
+ Parent("dir", "body");
+ Parent("div", "body");
+ Parent("dl", "body");
+ Parent("dt", "dl");
+ Parent("em", "body");
+ Parent("fieldset", "form");
+ Parent("font", "body");
+ Parent("form", "body");
+ Parent("frame", "frameset");
+ Parent("frameset", "html");
+ Parent("h1", "body");
+ Parent("h2", "body");
+ Parent("h3", "body");
+ Parent("h4", "body");
+ Parent("h5", "body");
+ Parent("h6", "body");
+ Parent("head", "html");
+ Parent("hr", "body");
+ Parent("i", "body");
+ Parent("iframe", "body");
+ Parent("img", "body");
+ Parent("input", "form");
+ Parent("ins", "body");
+ Parent("isindex", "head");
+ Parent("kbd", "body");
+ Parent("label", "form");
+ Parent("legend", "fieldset");
+ Parent("li", "ul");
+ Parent("link", "head");
+ Parent("listing", "body");
+ Parent("map", "body");
+ Parent("marquee", "body");
+ Parent("menu", "body");
+ Parent("meta", "head");
+ Parent("nobr", "body");
+ Parent("noframes", "html");
+ Parent("noscript", "body");
+ Parent("object", "body");
+ Parent("ol", "body");
+ Parent("optgroup", "select");
+ Parent("option", "select");
+ Parent("p", "body");
+ Parent("param", "object");
+ Parent("pre", "body");
+ Parent("q", "body");
+ Parent("rb", "body");
+ Parent("rbc", "body");
+ Parent("rp", "body");
+ Parent("rt", "body");
+ Parent("rtc", "body");
+ Parent("ruby", "body");
+ Parent("s", "body");
+ Parent("samp", "body");
+ Parent("script", "html");
+ Parent("select", "form");
+ Parent("small", "body");
+ Parent("span", "body");
+ Parent("strike", "body");
+ Parent("strong", "body");
+ Parent("style", "head");
+ Parent("sub", "body");
+ Parent("sup", "body");
+ Parent("table", "body");
+ Parent("tbody", "table");
+ Parent("td", "tr");
+ Parent("textarea", "form");
+ Parent("tfoot", "table");
+ Parent("th", "tr");
+ Parent("thead", "table");
+ Parent("title", "head");
+ Parent("tr", "tbody");
+ Parent("tt", "body");
+ Parent("u", "body");
+ Parent("ul", "body");
+ Parent("var", "body");
+ Parent("wbr", "body");
+ Parent("xmp", "body");
+ Attribute("a", "hreflang", "NMTOKEN", null);
+ Attribute("a", "shape", "CDATA", "rect");
+ Attribute("a", "tabindex", "NMTOKEN", null);
+ Attribute("applet", "align", "NMTOKEN", null);
+ Attribute("area", "nohref", "BOOLEAN", null);
+ Attribute("area", "shape", "CDATA", "rect");
+ Attribute("area", "tabindex", "NMTOKEN", null);
+ Attribute("br", "clear", "CDATA", "none");
+ Attribute("button", "disabled", "BOOLEAN", null);
+ Attribute("button", "tabindex", "NMTOKEN", null);
+ Attribute("button", "type", "CDATA", "submit");
+ Attribute("caption", "align", "NMTOKEN", null);
+ Attribute("col", "align", "NMTOKEN", null);
+ Attribute("col", "span", "CDATA", "1");
+ Attribute("col", "valign", "NMTOKEN", null);
+ Attribute("colgroup", "align", "NMTOKEN", null);
+ Attribute("colgroup", "span", "CDATA", "1");
+ Attribute("colgroup", "valign", "NMTOKEN", null);
+ Attribute("dir", "compact", "BOOLEAN", null);
+ Attribute("div", "align", "NMTOKEN", null);
+ Attribute("dl", "compact", "BOOLEAN", null);
+ Attribute("form", "enctype", "CDATA", "application/x-www-form-urlencoded");
+ Attribute("form", "method", "CDATA", "get");
+ Attribute("frame", "frameborder", "CDATA", "1");
+ Attribute("frame", "noresize", "BOOLEAN", null);
+ Attribute("frame", "scrolling", "CDATA", "auto");
+ Attribute("h1", "align", "NMTOKEN", null);
+ Attribute("h2", "align", "NMTOKEN", null);
+ Attribute("h3", "align", "NMTOKEN", null);
+ Attribute("h4", "align", "NMTOKEN", null);
+ Attribute("h5", "align", "NMTOKEN", null);
+ Attribute("h6", "align", "NMTOKEN", null);
+ Attribute("hr", "align", "NMTOKEN", null);
+ Attribute("hr", "noshade", "BOOLEAN", null);
+ Attribute("iframe", "align", "NMTOKEN", null);
+ Attribute("iframe", "frameborder", "CDATA", "1");
+ Attribute("iframe", "scrolling", "CDATA", "auto");
+ Attribute("img", "align", "NMTOKEN", null);
+ Attribute("img", "ismap", "BOOLEAN", null);
+ Attribute("input", "align", "NMTOKEN", null);
+ Attribute("input", "checked", "BOOLEAN", null);
+ Attribute("input", "disabled", "BOOLEAN", null);
+ Attribute("input", "ismap", "BOOLEAN", null);
+ Attribute("input", "maxlength", "NMTOKEN", null);
+ Attribute("input", "readonly", "BOOLEAN", null);
+ Attribute("input", "tabindex", "NMTOKEN", null);
+ Attribute("input", "type", "CDATA", "text");
+ Attribute("label", "for", "IDREF", null);
+ Attribute("legend", "align", "NMTOKEN", null);
+ Attribute("li", "value", "NMTOKEN", null);
+ Attribute("link", "hreflang", "NMTOKEN", null);
+ Attribute("marquee", "width", "NMTOKEN", null);
+ Attribute("menu", "compact", "BOOLEAN", null);
+ Attribute("meta", "http-equiv", "NMTOKEN", null);
+ Attribute("meta", "name", "NMTOKEN", null);
+ Attribute("object", "align", "NMTOKEN", null);
+ Attribute("object", "declare", "BOOLEAN", null);
+ Attribute("object", "tabindex", "NMTOKEN", null);
+ Attribute("ol", "compact", "BOOLEAN", null);
+ Attribute("ol", "start", "NMTOKEN", null);
+ Attribute("optgroup", "disabled", "BOOLEAN", null);
+ Attribute("option", "disabled", "BOOLEAN", null);
+ Attribute("option", "selected", "BOOLEAN", null);
+ Attribute("p", "align", "NMTOKEN", null);
+ Attribute("param", "valuetype", "CDATA", "data");
+ Attribute("pre", "width", "NMTOKEN", null);
+ Attribute("rt", "rbspan", "CDATA", "1");
+ Attribute("script", "defer", "BOOLEAN", null);
+ Attribute("select", "disabled", "BOOLEAN", null);
+ Attribute("select", "multiple", "BOOLEAN", null);
+ Attribute("select", "size", "NMTOKEN", null);
+ Attribute("select", "tabindex", "NMTOKEN", null);
+ Attribute("table", "align", "NMTOKEN", null);
+ Attribute("table", "frame", "NMTOKEN", null);
+ Attribute("table", "rules", "NMTOKEN", null);
+ Attribute("tbody", "align", "NMTOKEN", null);
+ Attribute("tbody", "valign", "NMTOKEN", null);
+ Attribute("td", "align", "NMTOKEN", null);
+ Attribute("td", "colspan", "CDATA", "1");
+ Attribute("td", "headers", "IDREFS", null);
+ Attribute("td", "nowrap", "BOOLEAN", null);
+ Attribute("td", "rowspan", "CDATA", "1");
+ Attribute("td", "scope", "NMTOKEN", null);
+ Attribute("td", "valign", "NMTOKEN", null);
+ Attribute("textarea", "cols", "NMTOKEN", null);
+ Attribute("textarea", "disabled", "BOOLEAN", null);
+ Attribute("textarea", "readonly", "BOOLEAN", null);
+ Attribute("textarea", "rows", "NMTOKEN", null);
+ Attribute("textarea", "tabindex", "NMTOKEN", null);
+ Attribute("tfoot", "align", "NMTOKEN", null);
+ Attribute("tfoot", "valign", "NMTOKEN", null);
+ Attribute("th", "align", "NMTOKEN", null);
+ Attribute("th", "colspan", "CDATA", "1");
+ Attribute("th", "headers", "IDREFS", null);
+ Attribute("th", "nowrap", "BOOLEAN", null);
+ Attribute("th", "rowspan", "CDATA", "1");
+ Attribute("th", "scope", "NMTOKEN", null);
+ Attribute("th", "valign", "NMTOKEN", null);
+ Attribute("thead", "align", "NMTOKEN", null);
+ Attribute("thead", "valign", "NMTOKEN", null);
+ Attribute("tr", "align", "NMTOKEN", null);
+ Attribute("tr", "valign", "NMTOKEN", null);
+ Attribute("ul", "compact", "BOOLEAN", null);
+ Attribute("ul", "type", "NMTOKEN", null);
+ Attribute("xmp", "width", "NMTOKEN", null);
+ Attribute("a", "class", "NMTOKEN", null);
+ Attribute("abbr", "class", "NMTOKEN", null);
+ Attribute("acronym", "class", "NMTOKEN", null);
+ Attribute("address", "class", "NMTOKEN", null);
+ Attribute("applet", "class", "NMTOKEN", null);
+ Attribute("area", "class", "NMTOKEN", null);
+ Attribute("b", "class", "NMTOKEN", null);
+ Attribute("base", "class", "NMTOKEN", null);
+ Attribute("basefont", "class", "NMTOKEN", null);
+ Attribute("bdo", "class", "NMTOKEN", null);
+ Attribute("bgsound", "class", "NMTOKEN", null);
+ Attribute("big", "class", "NMTOKEN", null);
+ Attribute("blink", "class", "NMTOKEN", null);
+ Attribute("blockquote", "class", "NMTOKEN", null);
+ Attribute("body", "class", "NMTOKEN", null);
+ Attribute("br", "class", "NMTOKEN", null);
+ Attribute("button", "class", "NMTOKEN", null);
+ Attribute("canvas", "class", "NMTOKEN", null);
+ Attribute("caption", "class", "NMTOKEN", null);
+ Attribute("center", "class", "NMTOKEN", null);
+ Attribute("cite", "class", "NMTOKEN", null);
+ Attribute("code", "class", "NMTOKEN", null);
+ Attribute("col", "class", "NMTOKEN", null);
+ Attribute("colgroup", "class", "NMTOKEN", null);
+ Attribute("comment", "class", "NMTOKEN", null);
+ Attribute("dd", "class", "NMTOKEN", null);
+ Attribute("del", "class", "NMTOKEN", null);
+ Attribute("dfn", "class", "NMTOKEN", null);
+ Attribute("dir", "class", "NMTOKEN", null);
+ Attribute("div", "class", "NMTOKEN", null);
+ Attribute("dl", "class", "NMTOKEN", null);
+ Attribute("dt", "class", "NMTOKEN", null);
+ Attribute("em", "class", "NMTOKEN", null);
+ Attribute("fieldset", "class", "NMTOKEN", null);
+ Attribute("font", "class", "NMTOKEN", null);
+ Attribute("form", "class", "NMTOKEN", null);
+ Attribute("frame", "class", "NMTOKEN", null);
+ Attribute("frameset", "class", "NMTOKEN", null);
+ Attribute("h1", "class", "NMTOKEN", null);
+ Attribute("h2", "class", "NMTOKEN", null);
+ Attribute("h3", "class", "NMTOKEN", null);
+ Attribute("h4", "class", "NMTOKEN", null);
+ Attribute("h5", "class", "NMTOKEN", null);
+ Attribute("h6", "class", "NMTOKEN", null);
+ Attribute("head", "class", "NMTOKEN", null);
+ Attribute("hr", "class", "NMTOKEN", null);
+ Attribute("html", "class", "NMTOKEN", null);
+ Attribute("i", "class", "NMTOKEN", null);
+ Attribute("iframe", "class", "NMTOKEN", null);
+ Attribute("img", "class", "NMTOKEN", null);
+ Attribute("input", "class", "NMTOKEN", null);
+ Attribute("ins", "class", "NMTOKEN", null);
+ Attribute("isindex", "class", "NMTOKEN", null);
+ Attribute("kbd", "class", "NMTOKEN", null);
+ Attribute("label", "class", "NMTOKEN", null);
+ Attribute("legend", "class", "NMTOKEN", null);
+ Attribute("li", "class", "NMTOKEN", null);
+ Attribute("link", "class", "NMTOKEN", null);
+ Attribute("listing", "class", "NMTOKEN", null);
+ Attribute("map", "class", "NMTOKEN", null);
+ Attribute("marquee", "class", "NMTOKEN", null);
+ Attribute("menu", "class", "NMTOKEN", null);
+ Attribute("meta", "class", "NMTOKEN", null);
+ Attribute("nobr", "class", "NMTOKEN", null);
+ Attribute("noframes", "class", "NMTOKEN", null);
+ Attribute("noscript", "class", "NMTOKEN", null);
+ Attribute("object", "class", "NMTOKEN", null);
+ Attribute("ol", "class", "NMTOKEN", null);
+ Attribute("optgroup", "class", "NMTOKEN", null);
+ Attribute("option", "class", "NMTOKEN", null);
+ Attribute("p", "class", "NMTOKEN", null);
+ Attribute("param", "class", "NMTOKEN", null);
+ Attribute("pre", "class", "NMTOKEN", null);
+ Attribute("q", "class", "NMTOKEN", null);
+ Attribute("rb", "class", "NMTOKEN", null);
+ Attribute("rbc", "class", "NMTOKEN", null);
+ Attribute("rp", "class", "NMTOKEN", null);
+ Attribute("rt", "class", "NMTOKEN", null);
+ Attribute("rtc", "class", "NMTOKEN", null);
+ Attribute("ruby", "class", "NMTOKEN", null);
+ Attribute("s", "class", "NMTOKEN", null);
+ Attribute("samp", "class", "NMTOKEN", null);
+ Attribute("script", "class", "NMTOKEN", null);
+ Attribute("select", "class", "NMTOKEN", null);
+ Attribute("small", "class", "NMTOKEN", null);
+ Attribute("span", "class", "NMTOKEN", null);
+ Attribute("strike", "class", "NMTOKEN", null);
+ Attribute("strong", "class", "NMTOKEN", null);
+ Attribute("style", "class", "NMTOKEN", null);
+ Attribute("sub", "class", "NMTOKEN", null);
+ Attribute("sup", "class", "NMTOKEN", null);
+ Attribute("table", "class", "NMTOKEN", null);
+ Attribute("tbody", "class", "NMTOKEN", null);
+ Attribute("td", "class", "NMTOKEN", null);
+ Attribute("textarea", "class", "NMTOKEN", null);
+ Attribute("tfoot", "class", "NMTOKEN", null);
+ Attribute("th", "class", "NMTOKEN", null);
+ Attribute("thead", "class", "NMTOKEN", null);
+ Attribute("title", "class", "NMTOKEN", null);
+ Attribute("tr", "class", "NMTOKEN", null);
+ Attribute("tt", "class", "NMTOKEN", null);
+ Attribute("u", "class", "NMTOKEN", null);
+ Attribute("ul", "class", "NMTOKEN", null);
+ Attribute("var", "class", "NMTOKEN", null);
+ Attribute("wbr", "class", "NMTOKEN", null);
+ Attribute("xmp", "class", "NMTOKEN", null);
+ Attribute("a", "dir", "NMTOKEN", null);
+ Attribute("abbr", "dir", "NMTOKEN", null);
+ Attribute("acronym", "dir", "NMTOKEN", null);
+ Attribute("address", "dir", "NMTOKEN", null);
+ Attribute("applet", "dir", "NMTOKEN", null);
+ Attribute("area", "dir", "NMTOKEN", null);
+ Attribute("b", "dir", "NMTOKEN", null);
+ Attribute("base", "dir", "NMTOKEN", null);
+ Attribute("basefont", "dir", "NMTOKEN", null);
+ Attribute("bdo", "dir", "NMTOKEN", null);
+ Attribute("bgsound", "dir", "NMTOKEN", null);
+ Attribute("big", "dir", "NMTOKEN", null);
+ Attribute("blink", "dir", "NMTOKEN", null);
+ Attribute("blockquote", "dir", "NMTOKEN", null);
+ Attribute("body", "dir", "NMTOKEN", null);
+ Attribute("br", "dir", "NMTOKEN", null);
+ Attribute("button", "dir", "NMTOKEN", null);
+ Attribute("canvas", "dir", "NMTOKEN", null);
+ Attribute("caption", "dir", "NMTOKEN", null);
+ Attribute("center", "dir", "NMTOKEN", null);
+ Attribute("cite", "dir", "NMTOKEN", null);
+ Attribute("code", "dir", "NMTOKEN", null);
+ Attribute("col", "dir", "NMTOKEN", null);
+ Attribute("colgroup", "dir", "NMTOKEN", null);
+ Attribute("comment", "dir", "NMTOKEN", null);
+ Attribute("dd", "dir", "NMTOKEN", null);
+ Attribute("del", "dir", "NMTOKEN", null);
+ Attribute("dfn", "dir", "NMTOKEN", null);
+ Attribute("dir", "dir", "NMTOKEN", null);
+ Attribute("div", "dir", "NMTOKEN", null);
+ Attribute("dl", "dir", "NMTOKEN", null);
+ Attribute("dt", "dir", "NMTOKEN", null);
+ Attribute("em", "dir", "NMTOKEN", null);
+ Attribute("fieldset", "dir", "NMTOKEN", null);
+ Attribute("font", "dir", "NMTOKEN", null);
+ Attribute("form", "dir", "NMTOKEN", null);
+ Attribute("frame", "dir", "NMTOKEN", null);
+ Attribute("frameset", "dir", "NMTOKEN", null);
+ Attribute("h1", "dir", "NMTOKEN", null);
+ Attribute("h2", "dir", "NMTOKEN", null);
+ Attribute("h3", "dir", "NMTOKEN", null);
+ Attribute("h4", "dir", "NMTOKEN", null);
+ Attribute("h5", "dir", "NMTOKEN", null);
+ Attribute("h6", "dir", "NMTOKEN", null);
+ Attribute("head", "dir", "NMTOKEN", null);
+ Attribute("hr", "dir", "NMTOKEN", null);
+ Attribute("html", "dir", "NMTOKEN", null);
+ Attribute("i", "dir", "NMTOKEN", null);
+ Attribute("iframe", "dir", "NMTOKEN", null);
+ Attribute("img", "dir", "NMTOKEN", null);
+ Attribute("input", "dir", "NMTOKEN", null);
+ Attribute("ins", "dir", "NMTOKEN", null);
+ Attribute("isindex", "dir", "NMTOKEN", null);
+ Attribute("kbd", "dir", "NMTOKEN", null);
+ Attribute("label", "dir", "NMTOKEN", null);
+ Attribute("legend", "dir", "NMTOKEN", null);
+ Attribute("li", "dir", "NMTOKEN", null);
+ Attribute("link", "dir", "NMTOKEN", null);
+ Attribute("listing", "dir", "NMTOKEN", null);
+ Attribute("map", "dir", "NMTOKEN", null);
+ Attribute("marquee", "dir", "NMTOKEN", null);
+ Attribute("menu", "dir", "NMTOKEN", null);
+ Attribute("meta", "dir", "NMTOKEN", null);
+ Attribute("nobr", "dir", "NMTOKEN", null);
+ Attribute("noframes", "dir", "NMTOKEN", null);
+ Attribute("noscript", "dir", "NMTOKEN", null);
+ Attribute("object", "dir", "NMTOKEN", null);
+ Attribute("ol", "dir", "NMTOKEN", null);
+ Attribute("optgroup", "dir", "NMTOKEN", null);
+ Attribute("option", "dir", "NMTOKEN", null);
+ Attribute("p", "dir", "NMTOKEN", null);
+ Attribute("param", "dir", "NMTOKEN", null);
+ Attribute("pre", "dir", "NMTOKEN", null);
+ Attribute("q", "dir", "NMTOKEN", null);
+ Attribute("rb", "dir", "NMTOKEN", null);
+ Attribute("rbc", "dir", "NMTOKEN", null);
+ Attribute("rp", "dir", "NMTOKEN", null);
+ Attribute("rt", "dir", "NMTOKEN", null);
+ Attribute("rtc", "dir", "NMTOKEN", null);
+ Attribute("ruby", "dir", "NMTOKEN", null);
+ Attribute("s", "dir", "NMTOKEN", null);
+ Attribute("samp", "dir", "NMTOKEN", null);
+ Attribute("script", "dir", "NMTOKEN", null);
+ Attribute("select", "dir", "NMTOKEN", null);
+ Attribute("small", "dir", "NMTOKEN", null);
+ Attribute("span", "dir", "NMTOKEN", null);
+ Attribute("strike", "dir", "NMTOKEN", null);
+ Attribute("strong", "dir", "NMTOKEN", null);
+ Attribute("style", "dir", "NMTOKEN", null);
+ Attribute("sub", "dir", "NMTOKEN", null);
+ Attribute("sup", "dir", "NMTOKEN", null);
+ Attribute("table", "dir", "NMTOKEN", null);
+ Attribute("tbody", "dir", "NMTOKEN", null);
+ Attribute("td", "dir", "NMTOKEN", null);
+ Attribute("textarea", "dir", "NMTOKEN", null);
+ Attribute("tfoot", "dir", "NMTOKEN", null);
+ Attribute("th", "dir", "NMTOKEN", null);
+ Attribute("thead", "dir", "NMTOKEN", null);
+ Attribute("title", "dir", "NMTOKEN", null);
+ Attribute("tr", "dir", "NMTOKEN", null);
+ Attribute("tt", "dir", "NMTOKEN", null);
+ Attribute("u", "dir", "NMTOKEN", null);
+ Attribute("ul", "dir", "NMTOKEN", null);
+ Attribute("var", "dir", "NMTOKEN", null);
+ Attribute("wbr", "dir", "NMTOKEN", null);
+ Attribute("xmp", "dir", "NMTOKEN", null);
+ Attribute("a", "id", "ID", null);
+ Attribute("abbr", "id", "ID", null);
+ Attribute("acronym", "id", "ID", null);
+ Attribute("address", "id", "ID", null);
+ Attribute("applet", "id", "ID", null);
+ Attribute("area", "id", "ID", null);
+ Attribute("b", "id", "ID", null);
+ Attribute("base", "id", "ID", null);
+ Attribute("basefont", "id", "ID", null);
+ Attribute("bdo", "id", "ID", null);
+ Attribute("bgsound", "id", "ID", null);
+ Attribute("big", "id", "ID", null);
+ Attribute("blink", "id", "ID", null);
+ Attribute("blockquote", "id", "ID", null);
+ Attribute("body", "id", "ID", null);
+ Attribute("br", "id", "ID", null);
+ Attribute("button", "id", "ID", null);
+ Attribute("canvas", "id", "ID", null);
+ Attribute("caption", "id", "ID", null);
+ Attribute("center", "id", "ID", null);
+ Attribute("cite", "id", "ID", null);
+ Attribute("code", "id", "ID", null);
+ Attribute("col", "id", "ID", null);
+ Attribute("colgroup", "id", "ID", null);
+ Attribute("comment", "id", "ID", null);
+ Attribute("dd", "id", "ID", null);
+ Attribute("del", "id", "ID", null);
+ Attribute("dfn", "id", "ID", null);
+ Attribute("dir", "id", "ID", null);
+ Attribute("div", "id", "ID", null);
+ Attribute("dl", "id", "ID", null);
+ Attribute("dt", "id", "ID", null);
+ Attribute("em", "id", "ID", null);
+ Attribute("fieldset", "id", "ID", null);
+ Attribute("font", "id", "ID", null);
+ Attribute("form", "id", "ID", null);
+ Attribute("frame", "id", "ID", null);
+ Attribute("frameset", "id", "ID", null);
+ Attribute("h1", "id", "ID", null);
+ Attribute("h2", "id", "ID", null);
+ Attribute("h3", "id", "ID", null);
+ Attribute("h4", "id", "ID", null);
+ Attribute("h5", "id", "ID", null);
+ Attribute("h6", "id", "ID", null);
+ Attribute("head", "id", "ID", null);
+ Attribute("hr", "id", "ID", null);
+ Attribute("html", "id", "ID", null);
+ Attribute("i", "id", "ID", null);
+ Attribute("iframe", "id", "ID", null);
+ Attribute("img", "id", "ID", null);
+ Attribute("input", "id", "ID", null);
+ Attribute("ins", "id", "ID", null);
+ Attribute("isindex", "id", "ID", null);
+ Attribute("kbd", "id", "ID", null);
+ Attribute("label", "id", "ID", null);
+ Attribute("legend", "id", "ID", null);
+ Attribute("li", "id", "ID", null);
+ Attribute("link", "id", "ID", null);
+ Attribute("listing", "id", "ID", null);
+ Attribute("map", "id", "ID", null);
+ Attribute("marquee", "id", "ID", null);
+ Attribute("menu", "id", "ID", null);
+ Attribute("meta", "id", "ID", null);
+ Attribute("nobr", "id", "ID", null);
+ Attribute("noframes", "id", "ID", null);
+ Attribute("noscript", "id", "ID", null);
+ Attribute("object", "id", "ID", null);
+ Attribute("ol", "id", "ID", null);
+ Attribute("optgroup", "id", "ID", null);
+ Attribute("option", "id", "ID", null);
+ Attribute("p", "id", "ID", null);
+ Attribute("param", "id", "ID", null);
+ Attribute("pre", "id", "ID", null);
+ Attribute("q", "id", "ID", null);
+ Attribute("rb", "id", "ID", null);
+ Attribute("rbc", "id", "ID", null);
+ Attribute("rp", "id", "ID", null);
+ Attribute("rt", "id", "ID", null);
+ Attribute("rtc", "id", "ID", null);
+ Attribute("ruby", "id", "ID", null);
+ Attribute("s", "id", "ID", null);
+ Attribute("samp", "id", "ID", null);
+ Attribute("script", "id", "ID", null);
+ Attribute("select", "id", "ID", null);
+ Attribute("small", "id", "ID", null);
+ Attribute("span", "id", "ID", null);
+ Attribute("strike", "id", "ID", null);
+ Attribute("strong", "id", "ID", null);
+ Attribute("style", "id", "ID", null);
+ Attribute("sub", "id", "ID", null);
+ Attribute("sup", "id", "ID", null);
+ Attribute("table", "id", "ID", null);
+ Attribute("tbody", "id", "ID", null);
+ Attribute("td", "id", "ID", null);
+ Attribute("textarea", "id", "ID", null);
+ Attribute("tfoot", "id", "ID", null);
+ Attribute("th", "id", "ID", null);
+ Attribute("thead", "id", "ID", null);
+ Attribute("title", "id", "ID", null);
+ Attribute("tr", "id", "ID", null);
+ Attribute("tt", "id", "ID", null);
+ Attribute("u", "id", "ID", null);
+ Attribute("ul", "id", "ID", null);
+ Attribute("var", "id", "ID", null);
+ Attribute("wbr", "id", "ID", null);
+ Attribute("xmp", "id", "ID", null);
+ Attribute("a", "lang", "NMTOKEN", null);
+ Attribute("abbr", "lang", "NMTOKEN", null);
+ Attribute("acronym", "lang", "NMTOKEN", null);
+ Attribute("address", "lang", "NMTOKEN", null);
+ Attribute("applet", "lang", "NMTOKEN", null);
+ Attribute("area", "lang", "NMTOKEN", null);
+ Attribute("b", "lang", "NMTOKEN", null);
+ Attribute("base", "lang", "NMTOKEN", null);
+ Attribute("basefont", "lang", "NMTOKEN", null);
+ Attribute("bdo", "lang", "NMTOKEN", null);
+ Attribute("bgsound", "lang", "NMTOKEN", null);
+ Attribute("big", "lang", "NMTOKEN", null);
+ Attribute("blink", "lang", "NMTOKEN", null);
+ Attribute("blockquote", "lang", "NMTOKEN", null);
+ Attribute("body", "lang", "NMTOKEN", null);
+ Attribute("br", "lang", "NMTOKEN", null);
+ Attribute("button", "lang", "NMTOKEN", null);
+ Attribute("canvas", "lang", "NMTOKEN", null);
+ Attribute("caption", "lang", "NMTOKEN", null);
+ Attribute("center", "lang", "NMTOKEN", null);
+ Attribute("cite", "lang", "NMTOKEN", null);
+ Attribute("code", "lang", "NMTOKEN", null);
+ Attribute("col", "lang", "NMTOKEN", null);
+ Attribute("colgroup", "lang", "NMTOKEN", null);
+ Attribute("comment", "lang", "NMTOKEN", null);
+ Attribute("dd", "lang", "NMTOKEN", null);
+ Attribute("del", "lang", "NMTOKEN", null);
+ Attribute("dfn", "lang", "NMTOKEN", null);
+ Attribute("dir", "lang", "NMTOKEN", null);
+ Attribute("div", "lang", "NMTOKEN", null);
+ Attribute("dl", "lang", "NMTOKEN", null);
+ Attribute("dt", "lang", "NMTOKEN", null);
+ Attribute("em", "lang", "NMTOKEN", null);
+ Attribute("fieldset", "lang", "NMTOKEN", null);
+ Attribute("font", "lang", "NMTOKEN", null);
+ Attribute("form", "lang", "NMTOKEN", null);
+ Attribute("frame", "lang", "NMTOKEN", null);
+ Attribute("frameset", "lang", "NMTOKEN", null);
+ Attribute("h1", "lang", "NMTOKEN", null);
+ Attribute("h2", "lang", "NMTOKEN", null);
+ Attribute("h3", "lang", "NMTOKEN", null);
+ Attribute("h4", "lang", "NMTOKEN", null);
+ Attribute("h5", "lang", "NMTOKEN", null);
+ Attribute("h6", "lang", "NMTOKEN", null);
+ Attribute("head", "lang", "NMTOKEN", null);
+ Attribute("hr", "lang", "NMTOKEN", null);
+ Attribute("html", "lang", "NMTOKEN", null);
+ Attribute("i", "lang", "NMTOKEN", null);
+ Attribute("iframe", "lang", "NMTOKEN", null);
+ Attribute("img", "lang", "NMTOKEN", null);
+ Attribute("input", "lang", "NMTOKEN", null);
+ Attribute("ins", "lang", "NMTOKEN", null);
+ Attribute("isindex", "lang", "NMTOKEN", null);
+ Attribute("kbd", "lang", "NMTOKEN", null);
+ Attribute("label", "lang", "NMTOKEN", null);
+ Attribute("legend", "lang", "NMTOKEN", null);
+ Attribute("li", "lang", "NMTOKEN", null);
+ Attribute("link", "lang", "NMTOKEN", null);
+ Attribute("listing", "lang", "NMTOKEN", null);
+ Attribute("map", "lang", "NMTOKEN", null);
+ Attribute("marquee", "lang", "NMTOKEN", null);
+ Attribute("menu", "lang", "NMTOKEN", null);
+ Attribute("meta", "lang", "NMTOKEN", null);
+ Attribute("nobr", "lang", "NMTOKEN", null);
+ Attribute("noframes", "lang", "NMTOKEN", null);
+ Attribute("noscript", "lang", "NMTOKEN", null);
+ Attribute("object", "lang", "NMTOKEN", null);
+ Attribute("ol", "lang", "NMTOKEN", null);
+ Attribute("optgroup", "lang", "NMTOKEN", null);
+ Attribute("option", "lang", "NMTOKEN", null);
+ Attribute("p", "lang", "NMTOKEN", null);
+ Attribute("param", "lang", "NMTOKEN", null);
+ Attribute("pre", "lang", "NMTOKEN", null);
+ Attribute("q", "lang", "NMTOKEN", null);
+ Attribute("rb", "lang", "NMTOKEN", null);
+ Attribute("rbc", "lang", "NMTOKEN", null);
+ Attribute("rp", "lang", "NMTOKEN", null);
+ Attribute("rt", "lang", "NMTOKEN", null);
+ Attribute("rtc", "lang", "NMTOKEN", null);
+ Attribute("ruby", "lang", "NMTOKEN", null);
+ Attribute("s", "lang", "NMTOKEN", null);
+ Attribute("samp", "lang", "NMTOKEN", null);
+ Attribute("script", "lang", "NMTOKEN", null);
+ Attribute("select", "lang", "NMTOKEN", null);
+ Attribute("small", "lang", "NMTOKEN", null);
+ Attribute("span", "lang", "NMTOKEN", null);
+ Attribute("strike", "lang", "NMTOKEN", null);
+ Attribute("strong", "lang", "NMTOKEN", null);
+ Attribute("style", "lang", "NMTOKEN", null);
+ Attribute("sub", "lang", "NMTOKEN", null);
+ Attribute("sup", "lang", "NMTOKEN", null);
+ Attribute("table", "lang", "NMTOKEN", null);
+ Attribute("tbody", "lang", "NMTOKEN", null);
+ Attribute("td", "lang", "NMTOKEN", null);
+ Attribute("textarea", "lang", "NMTOKEN", null);
+ Attribute("tfoot", "lang", "NMTOKEN", null);
+ Attribute("th", "lang", "NMTOKEN", null);
+ Attribute("thead", "lang", "NMTOKEN", null);
+ Attribute("title", "lang", "NMTOKEN", null);
+ Attribute("tr", "lang", "NMTOKEN", null);
+ Attribute("tt", "lang", "NMTOKEN", null);
+ Attribute("u", "lang", "NMTOKEN", null);
+ Attribute("ul", "lang", "NMTOKEN", null);
+ Attribute("var", "lang", "NMTOKEN", null);
+ Attribute("wbr", "lang", "NMTOKEN", null);
+ Attribute("xmp", "lang", "NMTOKEN", null);
+ Entity("aacgr", 0x03AC);
+ Entity("Aacgr", 0x0386);
+ Entity("aacute", 0x00E1);
+ Entity("Aacute", 0x00C1);
+ Entity("abreve", 0x0103);
+ Entity("Abreve", 0x0102);
+ Entity("ac", 0x223E);
+ Entity("acd", 0x223F);
+ Entity("acirc", 0x00E2);
+ Entity("Acirc", 0x00C2);
+ Entity("acute", 0x00B4);
+ Entity("acy", 0x0430);
+ Entity("Acy", 0x0410);
+ Entity("aelig", 0x00E6);
+ Entity("AElig", 0x00C6);
+ Entity("af", 0x2061);
+ Entity("afr", 0x1D51E);
+ Entity("Afr", 0x1D504);
+ Entity("agr", 0x03B1);
+ Entity("Agr", 0x0391);
+ Entity("agrave", 0x00E0);
+ Entity("Agrave", 0x00C0);
+ Entity("alefsym", 0x2135);
+ Entity("aleph", 0x2135);
+ Entity("alpha", 0x03B1);
+ Entity("Alpha", 0x0391);
+ Entity("amacr", 0x0101);
+ Entity("Amacr", 0x0100);
+ Entity("amalg", 0x2A3F);
+ Entity("amp", 0x0026);
+ Entity("and", 0x2227);
+ Entity("And", 0x2A53);
+ Entity("andand", 0x2A55);
+ Entity("andd", 0x2A5C);
+ Entity("andslope", 0x2A58);
+ Entity("andv", 0x2A5A);
+ Entity("ang", 0x2220);
+ Entity("ange", 0x29A4);
+ Entity("angle", 0x2220);
+ Entity("angmsd", 0x2221);
+ Entity("angmsdaa", 0x29A8);
+ Entity("angmsdab", 0x29A9);
+ Entity("angmsdac", 0x29AA);
+ Entity("angmsdad", 0x29AB);
+ Entity("angmsdae", 0x29AC);
+ Entity("angmsdaf", 0x29AD);
+ Entity("angmsdag", 0x29AE);
+ Entity("angmsdah", 0x29AF);
+ Entity("angrt", 0x221F);
+ Entity("angrtvb", 0x22BE);
+ Entity("angrtvbd", 0x299D);
+ Entity("angsph", 0x2222);
+ Entity("angst", 0x212B);
+ Entity("angzarr", 0x237C);
+ Entity("aogon", 0x0105);
+ Entity("Aogon", 0x0104);
+ Entity("aopf", 0x1D552);
+ Entity("Aopf", 0x1D538);
+ Entity("ap", 0x2248);
+ Entity("apacir", 0x2A6F);
+ Entity("ape", 0x224A);
+ Entity("apE", 0x2A70);
+ Entity("apid", 0x224B);
+ Entity("apos", 0x0027);
+ Entity("ApplyFunction", 0x2061);
+ Entity("approx", 0x2248);
+ Entity("approxeq", 0x224A);
+ Entity("aring", 0x00E5);
+ Entity("Aring", 0x00C5);
+ Entity("ascr", 0x1D4B6);
+ Entity("Ascr", 0x1D49C);
+ Entity("Assign", 0x2254);
+ Entity("ast", 0x002A);
+ Entity("asymp", 0x2248);
+ Entity("asympeq", 0x224D);
+ Entity("atilde", 0x00E3);
+ Entity("Atilde", 0x00C3);
+ Entity("auml", 0x00E4);
+ Entity("Auml", 0x00C4);
+ Entity("awconint", 0x2233);
+ Entity("awint", 0x2A11);
+ Entity("b.alpha", 0x1D6C2);
+ Entity("b.beta", 0x1D6C3);
+ Entity("b.chi", 0x1D6D8);
+ Entity("b.delta", 0x1D6C5);
+ Entity("b.Delta", 0x1D6AB);
+ Entity("b.epsi", 0x1D6C6);
+ Entity("b.epsiv", 0x1D6DC);
+ Entity("b.eta", 0x1D6C8);
+ Entity("b.gamma", 0x1D6C4);
+ Entity("b.Gamma", 0x1D6AA);
+ Entity("b.gammad", 0x1D7CB);
+ Entity("b.Gammad", 0x1D7CA);
+ Entity("b.iota", 0x1D6CA);
+ Entity("b.kappa", 0x1D6CB);
+ Entity("b.kappav", 0x1D6DE);
+ Entity("b.lambda", 0x1D6CC);
+ Entity("b.Lambda", 0x1D6B2);
+ Entity("b.mu", 0x1D6CD);
+ Entity("b.nu", 0x1D6CE);
+ Entity("b.omega", 0x1D6DA);
+ Entity("b.Omega", 0x1D6C0);
+ Entity("b.phi", 0x1D6D7);
+ Entity("b.Phi", 0x1D6BD);
+ Entity("b.phiv", 0x1D6DF);
+ Entity("b.pi", 0x1D6D1);
+ Entity("b.Pi", 0x1D6B7);
+ Entity("b.piv", 0x1D6E1);
+ Entity("b.psi", 0x1D6D9);
+ Entity("b.Psi", 0x1D6BF);
+ Entity("b.rho", 0x1D6D2);
+ Entity("b.rhov", 0x1D6E0);
+ Entity("b.sigma", 0x1D6D4);
+ Entity("b.Sigma", 0x1D6BA);
+ Entity("b.sigmav", 0x1D6D3);
+ Entity("b.tau", 0x1D6D5);
+ Entity("b.Theta", 0x1D6AF);
+ Entity("b.thetas", 0x1D6C9);
+ Entity("b.thetav", 0x1D6DD);
+ Entity("b.upsi", 0x1D6D6);
+ Entity("b.Upsi", 0x1D6BC);
+ Entity("b.xi", 0x1D6CF);
+ Entity("b.Xi", 0x1D6B5);
+ Entity("b.zeta", 0x1D6C7);
+ Entity("backcong", 0x224C);
+ Entity("backepsilon", 0x03F6);
+ Entity("backprime", 0x2035);
+ Entity("backsim", 0x223D);
+ Entity("backsimeq", 0x22CD);
+ Entity("Backslash", 0x2216);
+ Entity("Barv", 0x2AE7);
+ Entity("barvee", 0x22BD);
+ Entity("barwed", 0x2305);
+ Entity("Barwed", 0x2306);
+ Entity("barwedge", 0x2305);
+ Entity("bbrk", 0x23B5);
+ Entity("bbrktbrk", 0x23B6);
+ Entity("bcong", 0x224C);
+ Entity("bcy", 0x0431);
+ Entity("Bcy", 0x0411);
+ Entity("bdquo", 0x201E);
+ Entity("becaus", 0x2235);
+ Entity("because", 0x2235);
+ Entity("bemptyv", 0x29B0);
+ Entity("bepsi", 0x03F6);
+ Entity("bernou", 0x212C);
+ Entity("Bernoullis", 0x212C);
+ Entity("beta", 0x03B2);
+ Entity("Beta", 0x0392);
+ Entity("beth", 0x2136);
+ Entity("between", 0x226C);
+ Entity("bfr", 0x1D51F);
+ Entity("Bfr", 0x1D505);
+ Entity("bgr", 0x03B2);
+ Entity("Bgr", 0x0392);
+ Entity("bigcap", 0x22C2);
+ Entity("bigcirc", 0x25EF);
+ Entity("bigcup", 0x22C3);
+ Entity("bigodot", 0x2A00);
+ Entity("bigoplus", 0x2A01);
+ Entity("bigotimes", 0x2A02);
+ Entity("bigsqcup", 0x2A06);
+ Entity("bigstar", 0x2605);
+ Entity("bigtriangledown", 0x25BD);
+ Entity("bigtriangleup", 0x25B3);
+ Entity("biguplus", 0x2A04);
+ Entity("bigvee", 0x22C1);
+ Entity("bigwedge", 0x22C0);
+ Entity("bkarow", 0x290D);
+ Entity("blacklozenge", 0x29EB);
+ Entity("blacksquare", 0x25AA);
+ Entity("blacktriangle", 0x25B4);
+ Entity("blacktriangledown", 0x25BE);
+ Entity("blacktriangleleft", 0x25C2);
+ Entity("blacktriangleright", 0x25B8);
+ Entity("blank", 0x2423);
+ Entity("blk12", 0x2592);
+ Entity("blk14", 0x2591);
+ Entity("blk34", 0x2593);
+ Entity("block", 0x2588);
+ Entity("bnot", 0x2310);
+ Entity("bNot", 0x2AED);
+ Entity("bopf", 0x1D553);
+ Entity("Bopf", 0x1D539);
+ Entity("bot", 0x22A5);
+ Entity("bottom", 0x22A5);
+ Entity("bowtie", 0x22C8);
+ Entity("boxbox", 0x29C9);
+ Entity("boxdl", 0x2510);
+ Entity("boxdL", 0x2555);
+ Entity("boxDl", 0x2556);
+ Entity("boxDL", 0x2557);
+ Entity("boxdr", 0x250C);
+ Entity("boxdR", 0x2552);
+ Entity("boxDr", 0x2553);
+ Entity("boxDR", 0x2554);
+ Entity("boxh", 0x2500);
+ Entity("boxH", 0x2550);
+ Entity("boxhd", 0x252C);
+ Entity("boxhD", 0x2565);
+ Entity("boxHd", 0x2564);
+ Entity("boxHD", 0x2566);
+ Entity("boxhu", 0x2534);
+ Entity("boxhU", 0x2568);
+ Entity("boxHu", 0x2567);
+ Entity("boxHU", 0x2569);
+ Entity("boxminus", 0x229F);
+ Entity("boxplus", 0x229E);
+ Entity("boxtimes", 0x22A0);
+ Entity("boxul", 0x2518);
+ Entity("boxuL", 0x255B);
+ Entity("boxUl", 0x255C);
+ Entity("boxUL", 0x255D);
+ Entity("boxur", 0x2514);
+ Entity("boxuR", 0x2558);
+ Entity("boxUr", 0x2559);
+ Entity("boxUR", 0x255A);
+ Entity("boxv", 0x2502);
+ Entity("boxV", 0x2551);
+ Entity("boxvh", 0x253C);
+ Entity("boxvH", 0x256A);
+ Entity("boxVh", 0x256B);
+ Entity("boxVH", 0x256C);
+ Entity("boxvl", 0x2524);
+ Entity("boxvL", 0x2561);
+ Entity("boxVl", 0x2562);
+ Entity("boxVL", 0x2563);
+ Entity("boxvr", 0x251C);
+ Entity("boxvR", 0x255E);
+ Entity("boxVr", 0x255F);
+ Entity("boxVR", 0x2560);
+ Entity("bprime", 0x2035);
+ Entity("breve", 0x02D8);
+ Entity("brvbar", 0x00A6);
+ Entity("bscr", 0x1D4B7);
+ Entity("Bscr", 0x212C);
+ Entity("bsemi", 0x204F);
+ Entity("bsim", 0x223D);
+ Entity("bsime", 0x22CD);
+ Entity("bsol", 0x005C);
+ Entity("bsolb", 0x29C5);
+ Entity("bull", 0x2022);
+ Entity("bullet", 0x2022);
+ Entity("bump", 0x224E);
+ Entity("bumpe", 0x224F);
+ Entity("bumpE", 0x2AAE);
+ Entity("bumpeq", 0x224F);
+ Entity("Bumpeq", 0x224E);
+ Entity("cacute", 0x0107);
+ Entity("Cacute", 0x0106);
+ Entity("cap", 0x2229);
+ Entity("Cap", 0x22D2);
+ Entity("capand", 0x2A44);
+ Entity("capbrcup", 0x2A49);
+ Entity("capcap", 0x2A4B);
+ Entity("capcup", 0x2A47);
+ Entity("capdot", 0x2A40);
+ Entity("CapitalDifferentialD", 0x2145);
+ Entity("caret", 0x2041);
+ Entity("caron", 0x02C7);
+ Entity("Cayleys", 0x212D);
+ Entity("ccaps", 0x2A4D);
+ Entity("ccaron", 0x010D);
+ Entity("Ccaron", 0x010C);
+ Entity("ccedil", 0x00E7);
+ Entity("Ccedil", 0x00C7);
+ Entity("ccirc", 0x0109);
+ Entity("Ccirc", 0x0108);
+ Entity("Cconint", 0x2230);
+ Entity("ccups", 0x2A4C);
+ Entity("ccupssm", 0x2A50);
+ Entity("cdot", 0x010B);
+ Entity("Cdot", 0x010A);
+ Entity("cedil", 0x00B8);
+ Entity("Cedilla", 0x00B8);
+ Entity("cemptyv", 0x29B2);
+ Entity("cent", 0x00A2);
+ Entity("centerdot", 0x00B7);
+ Entity("cfr", 0x1D520);
+ Entity("Cfr", 0x212D);
+ Entity("chcy", 0x0447);
+ Entity("CHcy", 0x0427);
+ Entity("check", 0x2713);
+ Entity("checkmark", 0x2713);
+ Entity("chi", 0x03C7);
+ Entity("Chi", 0x03A7);
+ Entity("cir", 0x25CB);
+ Entity("circ", 0x02C6);
+ Entity("circeq", 0x2257);
+ Entity("circlearrowleft", 0x21BA);
+ Entity("circlearrowright", 0x21BB);
+ Entity("circledast", 0x229B);
+ Entity("circledcirc", 0x229A);
+ Entity("circleddash", 0x229D);
+ Entity("CircleDot", 0x2299);
+ Entity("circledR", 0x00AE);
+ Entity("circledS", 0x24C8);
+ Entity("CircleMinus", 0x2296);
+ Entity("CirclePlus", 0x2295);
+ Entity("CircleTimes", 0x2297);
+ Entity("cire", 0x2257);
+ Entity("cirE", 0x29C3);
+ Entity("cirfnint", 0x2A10);
+ Entity("cirmid", 0x2AEF);
+ Entity("cirscir", 0x29C2);
+ Entity("ClockwiseContourIntegral", 0x2232);
+ Entity("CloseCurlyDoubleQuote", 0x201D);
+ Entity("CloseCurlyQuote", 0x2019);
+ Entity("clubs", 0x2663);
+ Entity("clubsuit", 0x2663);
+ Entity("colon", 0x003A);
+ Entity("Colon", 0x2237);
+ Entity("colone", 0x2254);
+ Entity("Colone", 0x2A74);
+ Entity("coloneq", 0x2254);
+ Entity("comma", 0x002C);
+ Entity("commat", 0x0040);
+ Entity("comp", 0x2201);
+ Entity("compfn", 0x2218);
+ Entity("complement", 0x2201);
+ Entity("complexes", 0x2102);
+ Entity("cong", 0x2245);
+ Entity("congdot", 0x2A6D);
+ Entity("Congruent", 0x2261);
+ Entity("conint", 0x222E);
+ Entity("Conint", 0x222F);
+ Entity("ContourIntegral", 0x222E);
+ Entity("copf", 0x1D554);
+ Entity("Copf", 0x2102);
+ Entity("coprod", 0x2210);
+ Entity("Coproduct", 0x2210);
+ Entity("copy", 0x00A9);
+ Entity("copysr", 0x2117);
+ Entity("CounterClockwiseContourIntegral", 0x2233);
+ Entity("crarr", 0x21B5);
+ Entity("cross", 0x2717);
+ Entity("Cross", 0x2A2F);
+ Entity("cscr", 0x1D4B8);
+ Entity("Cscr", 0x1D49E);
+ Entity("csub", 0x2ACF);
+ Entity("csube", 0x2AD1);
+ Entity("csup", 0x2AD0);
+ Entity("csupe", 0x2AD2);
+ Entity("ctdot", 0x22EF);
+ Entity("cudarrl", 0x2938);
+ Entity("cudarrr", 0x2935);
+ Entity("cuepr", 0x22DE);
+ Entity("cuesc", 0x22DF);
+ Entity("cularr", 0x21B6);
+ Entity("cularrp", 0x293D);
+ Entity("cup", 0x222A);
+ Entity("Cup", 0x22D3);
+ Entity("cupbrcap", 0x2A48);
+ Entity("cupcap", 0x2A46);
+ Entity("CupCap", 0x224D);
+ Entity("cupcup", 0x2A4A);
+ Entity("cupdot", 0x228D);
+ Entity("cupor", 0x2A45);
+ Entity("curarr", 0x21B7);
+ Entity("curarrm", 0x293C);
+ Entity("curlyeqprec", 0x22DE);
+ Entity("curlyeqsucc", 0x22DF);
+ Entity("curlyvee", 0x22CE);
+ Entity("curlywedge", 0x22CF);
+ Entity("curren", 0x00A4);
+ Entity("curvearrowleft", 0x21B6);
+ Entity("curvearrowright", 0x21B7);
+ Entity("cuvee", 0x22CE);
+ Entity("cuwed", 0x22CF);
+ Entity("cwconint", 0x2232);
+ Entity("cwint", 0x2231);
+ Entity("cylcty", 0x232D);
+ Entity("dagger", 0x2020);
+ Entity("Dagger", 0x2021);
+ Entity("daleth", 0x2138);
+ Entity("darr", 0x2193);
+ Entity("dArr", 0x21D3);
+ Entity("Darr", 0x21A1);
+ Entity("dash", 0x2010);
+ Entity("dashv", 0x22A3);
+ Entity("Dashv", 0x2AE4);
+ Entity("dbkarow", 0x290F);
+ Entity("dblac", 0x02DD);
+ Entity("dcaron", 0x010F);
+ Entity("Dcaron", 0x010E);
+ Entity("dcy", 0x0434);
+ Entity("Dcy", 0x0414);
+ Entity("dd", 0x2146);
+ Entity("DD", 0x2145);
+ Entity("ddagger", 0x2021);
+ Entity("ddarr", 0x21CA);
+ Entity("DDotrahd", 0x2911);
+ Entity("ddotseq", 0x2A77);
+ Entity("deg", 0x00B0);
+ Entity("Del", 0x2207);
+ Entity("delta", 0x03B4);
+ Entity("Delta", 0x0394);
+ Entity("demptyv", 0x29B1);
+ Entity("dfisht", 0x297F);
+ Entity("dfr", 0x1D521);
+ Entity("Dfr", 0x1D507);
+ Entity("dgr", 0x03B4);
+ Entity("Dgr", 0x0394);
+ Entity("dHar", 0x2965);
+ Entity("dharl", 0x21C3);
+ Entity("dharr", 0x21C2);
+ Entity("DiacriticalAcute", 0x00B4);
+ Entity("DiacriticalDot", 0x02D9);
+ Entity("DiacriticalDoubleAcute", 0x02DD);
+ Entity("DiacriticalGrave", 0x0060);
+ Entity("DiacriticalTilde", 0x02DC);
+ Entity("diam", 0x22C4);
+ Entity("diamond", 0x22C4);
+ Entity("diamondsuit", 0x2666);
+ Entity("diams", 0x2666);
+ Entity("die", 0x00A8);
+ Entity("DifferentialD", 0x2146);
+ Entity("digamma", 0x03DD);
+ Entity("disin", 0x22F2);
+ Entity("div", 0x00F7);
+ Entity("divide", 0x00F7);
+ Entity("divideontimes", 0x22C7);
+ Entity("divonx", 0x22C7);
+ Entity("djcy", 0x0452);
+ Entity("DJcy", 0x0402);
+ Entity("dlcorn", 0x231E);
+ Entity("dlcrop", 0x230D);
+ Entity("dollar", 0x0024);
+ Entity("dopf", 0x1D555);
+ Entity("Dopf", 0x1D53B);
+ Entity("dot", 0x02D9);
+ Entity("Dot", 0x00A8);
+ Entity("doteq", 0x2250);
+ Entity("doteqdot", 0x2251);
+ Entity("DotEqual", 0x2250);
+ Entity("dotminus", 0x2238);
+ Entity("dotplus", 0x2214);
+ Entity("dotsquare", 0x22A1);
+ Entity("doublebarwedge", 0x2306);
+ Entity("DoubleContourIntegral", 0x222F);
+ Entity("DoubleDot", 0x00A8);
+ Entity("DoubleDownArrow", 0x21D3);
+ Entity("DoubleLeftArrow", 0x21D0);
+ Entity("DoubleLeftRightArrow", 0x21D4);
+ Entity("DoubleLeftTee", 0x2AE4);
+ Entity("DoubleLongLeftArrow", 0x27F8);
+ Entity("DoubleLongLeftRightArrow", 0x27FA);
+ Entity("DoubleLongRightArrow", 0x27F9);
+ Entity("DoubleRightArrow", 0x21D2);
+ Entity("DoubleRightTee", 0x22A8);
+ Entity("DoubleUpArrow", 0x21D1);
+ Entity("DoubleUpDownArrow", 0x21D5);
+ Entity("DoubleVerticalBar", 0x2225);
+ Entity("downarrow", 0x2193);
+ Entity("Downarrow", 0x21D3);
+ Entity("DownArrowBar", 0x2913);
+ Entity("DownArrowUpArrow", 0x21F5);
+ Entity("downdownarrows", 0x21CA);
+ Entity("downharpoonleft", 0x21C3);
+ Entity("downharpoonright", 0x21C2);
+ Entity("DownLeftRightVector", 0x2950);
+ Entity("DownLeftTeeVector", 0x295E);
+ Entity("DownLeftVector", 0x21BD);
+ Entity("DownLeftVectorBar", 0x2956);
+ Entity("DownRightTeeVector", 0x295F);
+ Entity("DownRightVector", 0x21C1);
+ Entity("DownRightVectorBar", 0x2957);
+ Entity("DownTee", 0x22A4);
+ Entity("DownTeeArrow", 0x21A7);
+ Entity("drbkarow", 0x2910);
+ Entity("drcorn", 0x231F);
+ Entity("drcrop", 0x230C);
+ Entity("dscr", 0x1D4B9);
+ Entity("Dscr", 0x1D49F);
+ Entity("dscy", 0x0455);
+ Entity("DScy", 0x0405);
+ Entity("dsol", 0x29F6);
+ Entity("dstrok", 0x0111);
+ Entity("Dstrok", 0x0110);
+ Entity("dtdot", 0x22F1);
+ Entity("dtri", 0x25BF);
+ Entity("dtrif", 0x25BE);
+ Entity("duarr", 0x21F5);
+ Entity("duhar", 0x296F);
+ Entity("dwangle", 0x29A6);
+ Entity("dzcy", 0x045F);
+ Entity("DZcy", 0x040F);
+ Entity("dzigrarr", 0x27FF);
+ Entity("eacgr", 0x03AD);
+ Entity("Eacgr", 0x0388);
+ Entity("eacute", 0x00E9);
+ Entity("Eacute", 0x00C9);
+ Entity("easter", 0x2A6E);
+ Entity("ecaron", 0x011B);
+ Entity("Ecaron", 0x011A);
+ Entity("ecir", 0x2256);
+ Entity("ecirc", 0x00EA);
+ Entity("Ecirc", 0x00CA);
+ Entity("ecolon", 0x2255);
+ Entity("ecy", 0x044D);
+ Entity("Ecy", 0x042D);
+ Entity("eDDot", 0x2A77);
+ Entity("edot", 0x0117);
+ Entity("eDot", 0x2251);
+ Entity("Edot", 0x0116);
+ Entity("ee", 0x2147);
+ Entity("eeacgr", 0x03AE);
+ Entity("EEacgr", 0x0389);
+ Entity("eegr", 0x03B7);
+ Entity("EEgr", 0x0397);
+ Entity("efDot", 0x2252);
+ Entity("efr", 0x1D522);
+ Entity("Efr", 0x1D508);
+ Entity("eg", 0x2A9A);
+ Entity("egr", 0x03B5);
+ Entity("Egr", 0x0395);
+ Entity("egrave", 0x00E8);
+ Entity("Egrave", 0x00C8);
+ Entity("egs", 0x2A96);
+ Entity("egsdot", 0x2A98);
+ Entity("el", 0x2A99);
+ Entity("Element", 0x2208);
+ Entity("elinters", 0x23E7);
+ Entity("ell", 0x2113);
+ Entity("els", 0x2A95);
+ Entity("elsdot", 0x2A97);
+ Entity("emacr", 0x0113);
+ Entity("Emacr", 0x0112);
+ Entity("empty", 0x2205);
+ Entity("emptyset", 0x2205);
+ Entity("EmptySmallSquare", 0x25FB);
+ Entity("emptyv", 0x2205);
+ Entity("EmptyVerySmallSquare", 0x25AB);
+ Entity("emsp", 0x2003);
+ Entity("emsp13", 0x2004);
+ Entity("emsp14", 0x2005);
+ Entity("eng", 0x014B);
+ Entity("ENG", 0x014A);
+ Entity("ensp", 0x2002);
+ Entity("eogon", 0x0119);
+ Entity("Eogon", 0x0118);
+ Entity("eopf", 0x1D556);
+ Entity("Eopf", 0x1D53C);
+ Entity("epar", 0x22D5);
+ Entity("eparsl", 0x29E3);
+ Entity("eplus", 0x2A71);
+ Entity("epsi", 0x03F5);
+ Entity("epsilon", 0x03B5);
+ Entity("Epsilon", 0x0395);
+ Entity("epsiv", 0x03B5);
+ Entity("eqcirc", 0x2256);
+ Entity("eqcolon", 0x2255);
+ Entity("eqsim", 0x2242);
+ Entity("eqslantgtr", 0x2A96);
+ Entity("eqslantless", 0x2A95);
+ Entity("Equal", 0x2A75);
+ Entity("equals", 0x003D);
+ Entity("EqualTilde", 0x2242);
+ Entity("equest", 0x225F);
+ Entity("Equilibrium", 0x21CC);
+ Entity("equiv", 0x2261);
+ Entity("equivDD", 0x2A78);
+ Entity("eqvparsl", 0x29E5);
+ Entity("erarr", 0x2971);
+ Entity("erDot", 0x2253);
+ Entity("escr", 0x212F);
+ Entity("Escr", 0x2130);
+ Entity("esdot", 0x2250);
+ Entity("esim", 0x2242);
+ Entity("Esim", 0x2A73);
+ Entity("eta", 0x03B7);
+ Entity("Eta", 0x0397);
+ Entity("eth", 0x00F0);
+ Entity("ETH", 0x00D0);
+ Entity("euml", 0x00EB);
+ Entity("Euml", 0x00CB);
+ Entity("euro", 0x20AC);
+ Entity("excl", 0x0021);
+ Entity("exist", 0x2203);
+ Entity("Exists", 0x2203);
+ Entity("expectation", 0x2130);
+ Entity("exponentiale", 0x2147);
+ Entity("fallingdotseq", 0x2252);
+ Entity("fcy", 0x0444);
+ Entity("Fcy", 0x0424);
+ Entity("female", 0x2640);
+ Entity("ffilig", 0xFB03);
+ Entity("fflig", 0xFB00);
+ Entity("ffllig", 0xFB04);
+ Entity("ffr", 0x1D523);
+ Entity("Ffr", 0x1D509);
+ Entity("filig", 0xFB01);
+ Entity("FilledSmallSquare", 0x25FC);
+ Entity("FilledVerySmallSquare", 0x25AA);
+ Entity("flat", 0x266D);
+ Entity("fllig", 0xFB02);
+ Entity("fltns", 0x25B1);
+ Entity("fnof", 0x0192);
+ Entity("fopf", 0x1D557);
+ Entity("Fopf", 0x1D53D);
+ Entity("forall", 0x2200);
+ Entity("fork", 0x22D4);
+ Entity("forkv", 0x2AD9);
+ Entity("Fouriertrf", 0x2131);
+ Entity("fpartint", 0x2A0D);
+ Entity("frac12", 0x00BD);
+ Entity("frac13", 0x2153);
+ Entity("frac14", 0x00BC);
+ Entity("frac15", 0x2155);
+ Entity("frac16", 0x2159);
+ Entity("frac18", 0x215B);
+ Entity("frac23", 0x2154);
+ Entity("frac25", 0x2156);
+ Entity("frac34", 0x00BE);
+ Entity("frac35", 0x2157);
+ Entity("frac38", 0x215C);
+ Entity("frac45", 0x2158);
+ Entity("frac56", 0x215A);
+ Entity("frac58", 0x215D);
+ Entity("frac78", 0x215E);
+ Entity("frasl", 0x2044);
+ Entity("frown", 0x2322);
+ Entity("fscr", 0x1D4BB);
+ Entity("Fscr", 0x2131);
+ Entity("gacute", 0x01F5);
+ Entity("gamma", 0x03B3);
+ Entity("Gamma", 0x0393);
+ Entity("gammad", 0x03DD);
+ Entity("Gammad", 0x03DC);
+ Entity("gap", 0x2A86);
+ Entity("gbreve", 0x011F);
+ Entity("Gbreve", 0x011E);
+ Entity("Gcedil", 0x0122);
+ Entity("gcirc", 0x011D);
+ Entity("Gcirc", 0x011C);
+ Entity("gcy", 0x0433);
+ Entity("Gcy", 0x0413);
+ Entity("gdot", 0x0121);
+ Entity("Gdot", 0x0120);
+ Entity("ge", 0x2265);
+ Entity("gE", 0x2267);
+ Entity("gel", 0x22DB);
+ Entity("gEl", 0x2A8C);
+ Entity("geq", 0x2265);
+ Entity("geqq", 0x2267);
+ Entity("geqslant", 0x2A7E);
+ Entity("ges", 0x2A7E);
+ Entity("gescc", 0x2AA9);
+ Entity("gesdot", 0x2A80);
+ Entity("gesdoto", 0x2A82);
+ Entity("gesdotol", 0x2A84);
+ Entity("gesles", 0x2A94);
+ Entity("gfr", 0x1D524);
+ Entity("Gfr", 0x1D50A);
+ Entity("gg", 0x226B);
+ Entity("Gg", 0x22D9);
+ Entity("ggg", 0x22D9);
+ Entity("ggr", 0x03B3);
+ Entity("Ggr", 0x0393);
+ Entity("gimel", 0x2137);
+ Entity("gjcy", 0x0453);
+ Entity("GJcy", 0x0403);
+ Entity("gl", 0x2277);
+ Entity("gla", 0x2AA5);
+ Entity("glE", 0x2A92);
+ Entity("glj", 0x2AA4);
+ Entity("gnap", 0x2A8A);
+ Entity("gnapprox", 0x2A8A);
+ Entity("gne", 0x2A88);
+ Entity("gnE", 0x2269);
+ Entity("gneq", 0x2A88);
+ Entity("gneqq", 0x2269);
+ Entity("gnsim", 0x22E7);
+ Entity("gopf", 0x1D558);
+ Entity("Gopf", 0x1D53E);
+ Entity("grave", 0x0060);
+ Entity("GreaterEqual", 0x2265);
+ Entity("GreaterEqualLess", 0x22DB);
+ Entity("GreaterFullEqual", 0x2267);
+ Entity("GreaterGreater", 0x2AA2);
+ Entity("GreaterLess", 0x2277);
+ Entity("GreaterSlantEqual", 0x2A7E);
+ Entity("GreaterTilde", 0x2273);
+ Entity("gscr", 0x210A);
+ Entity("Gscr", 0x1D4A2);
+ Entity("gsim", 0x2273);
+ Entity("gsime", 0x2A8E);
+ Entity("gsiml", 0x2A90);
+ Entity("gt", 0x003E);
+ Entity("Gt", 0x226B);
+ Entity("gtcc", 0x2AA7);
+ Entity("gtcir", 0x2A7A);
+ Entity("gtdot", 0x22D7);
+ Entity("gtlPar", 0x2995);
+ Entity("gtquest", 0x2A7C);
+ Entity("gtrapprox", 0x2A86);
+ Entity("gtrarr", 0x2978);
+ Entity("gtrdot", 0x22D7);
+ Entity("gtreqless", 0x22DB);
+ Entity("gtreqqless", 0x2A8C);
+ Entity("gtrless", 0x2277);
+ Entity("gtrsim", 0x2273);
+ Entity("Hacek", 0x02C7);
+ Entity("hairsp", 0x200A);
+ Entity("half", 0x00BD);
+ Entity("hamilt", 0x210B);
+ Entity("hardcy", 0x044A);
+ Entity("HARDcy", 0x042A);
+ Entity("harr", 0x2194);
+ Entity("hArr", 0x21D4);
+ Entity("harrcir", 0x2948);
+ Entity("harrw", 0x21AD);
+ Entity("Hat", 0x005E);
+ Entity("hbar", 0x210F);
+ Entity("hcirc", 0x0125);
+ Entity("Hcirc", 0x0124);
+ Entity("hearts", 0x2665);
+ Entity("heartsuit", 0x2665);
+ Entity("hellip", 0x2026);
+ Entity("hercon", 0x22B9);
+ Entity("hfr", 0x1D525);
+ Entity("Hfr", 0x210C);
+ Entity("HilbertSpace", 0x210B);
+ Entity("hksearow", 0x2925);
+ Entity("hkswarow", 0x2926);
+ Entity("hoarr", 0x21FF);
+ Entity("homtht", 0x223B);
+ Entity("hookleftarrow", 0x21A9);
+ Entity("hookrightarrow", 0x21AA);
+ Entity("hopf", 0x1D559);
+ Entity("Hopf", 0x210D);
+ Entity("horbar", 0x2015);
+ Entity("HorizontalLine", 0x2500);
+ Entity("hscr", 0x1D4BD);
+ Entity("Hscr", 0x210B);
+ Entity("hslash", 0x210F);
+ Entity("hstrok", 0x0127);
+ Entity("Hstrok", 0x0126);
+ Entity("HumpDownHump", 0x224E);
+ Entity("HumpEqual", 0x224F);
+ Entity("hybull", 0x2043);
+ Entity("hyphen", 0x2010);
+ Entity("iacgr", 0x03AF);
+ Entity("Iacgr", 0x038A);
+ Entity("iacute", 0x00ED);
+ Entity("Iacute", 0x00CD);
+ Entity("ic", 0x2063);
+ Entity("icirc", 0x00EE);
+ Entity("Icirc", 0x00CE);
+ Entity("icy", 0x0438);
+ Entity("Icy", 0x0418);
+ Entity("idiagr", 0x0390);
+ Entity("idigr", 0x03CA);
+ Entity("Idigr", 0x03AA);
+ Entity("Idot", 0x0130);
+ Entity("iecy", 0x0435);
+ Entity("IEcy", 0x0415);
+ Entity("iexcl", 0x00A1);
+ Entity("iff", 0x21D4);
+ Entity("ifr", 0x1D526);
+ Entity("Ifr", 0x2111);
+ Entity("igr", 0x03B9);
+ Entity("Igr", 0x0399);
+ Entity("igrave", 0x00EC);
+ Entity("Igrave", 0x00CC);
+ Entity("ii", 0x2148);
+ Entity("iiiint", 0x2A0C);
+ Entity("iiint", 0x222D);
+ Entity("iinfin", 0x29DC);
+ Entity("iiota", 0x2129);
+ Entity("ijlig", 0x0133);
+ Entity("IJlig", 0x0132);
+ Entity("Im", 0x2111);
+ Entity("imacr", 0x012B);
+ Entity("Imacr", 0x012A);
+ Entity("image", 0x2111);
+ Entity("ImaginaryI", 0x2148);
+ Entity("imagline", 0x2110);
+ Entity("imagpart", 0x2111);
+ Entity("imath", 0x0131);
+ Entity("imof", 0x22B7);
+ Entity("imped", 0x01B5);
+ Entity("Implies", 0x21D2);
+ Entity("in", 0x2208);
+ Entity("incare", 0x2105);
+ Entity("infin", 0x221E);
+ Entity("infintie", 0x29DD);
+ Entity("inodot", 0x0131);
+ Entity("int", 0x222B);
+ Entity("Int", 0x222C);
+ Entity("intcal", 0x22BA);
+ Entity("integers", 0x2124);
+ Entity("Integral", 0x222B);
+ Entity("intercal", 0x22BA);
+ Entity("Intersection", 0x22C2);
+ Entity("intlarhk", 0x2A17);
+ Entity("intprod", 0x2A3C);
+ Entity("InvisibleComma", 0x2063);
+ Entity("InvisibleTimes", 0x2062);
+ Entity("iocy", 0x0451);
+ Entity("IOcy", 0x0401);
+ Entity("iogon", 0x012F);
+ Entity("Iogon", 0x012E);
+ Entity("iopf", 0x1D55A);
+ Entity("Iopf", 0x1D540);
+ Entity("iota", 0x03B9);
+ Entity("Iota", 0x0399);
+ Entity("iprod", 0x2A3C);
+ Entity("iquest", 0x00BF);
+ Entity("iscr", 0x1D4BE);
+ Entity("Iscr", 0x2110);
+ Entity("isin", 0x2208);
+ Entity("isindot", 0x22F5);
+ Entity("isinE", 0x22F9);
+ Entity("isins", 0x22F4);
+ Entity("isinsv", 0x22F3);
+ Entity("isinv", 0x2208);
+ Entity("it", 0x2062);
+ Entity("itilde", 0x0129);
+ Entity("Itilde", 0x0128);
+ Entity("iukcy", 0x0456);
+ Entity("Iukcy", 0x0406);
+ Entity("iuml", 0x00EF);
+ Entity("Iuml", 0x00CF);
+ Entity("jcirc", 0x0135);
+ Entity("Jcirc", 0x0134);
+ Entity("jcy", 0x0439);
+ Entity("Jcy", 0x0419);
+ Entity("jfr", 0x1D527);
+ Entity("Jfr", 0x1D50D);
+ Entity("jmath", 0x0237);
+ Entity("jopf", 0x1D55B);
+ Entity("Jopf", 0x1D541);
+ Entity("jscr", 0x1D4BF);
+ Entity("Jscr", 0x1D4A5);
+ Entity("jsercy", 0x0458);
+ Entity("Jsercy", 0x0408);
+ Entity("jukcy", 0x0454);
+ Entity("Jukcy", 0x0404);
+ Entity("kappa", 0x03BA);
+ Entity("Kappa", 0x039A);
+ Entity("kappav", 0x03F0);
+ Entity("kcedil", 0x0137);
+ Entity("Kcedil", 0x0136);
+ Entity("kcy", 0x043A);
+ Entity("Kcy", 0x041A);
+ Entity("kfr", 0x1D528);
+ Entity("Kfr", 0x1D50E);
+ Entity("kgr", 0x03BA);
+ Entity("Kgr", 0x039A);
+ Entity("kgreen", 0x0138);
+ Entity("khcy", 0x0445);
+ Entity("KHcy", 0x0425);
+ Entity("khgr", 0x03C7);
+ Entity("KHgr", 0x03A7);
+ Entity("kjcy", 0x045C);
+ Entity("KJcy", 0x040C);
+ Entity("kopf", 0x1D55C);
+ Entity("Kopf", 0x1D542);
+ Entity("kscr", 0x1D4C0);
+ Entity("Kscr", 0x1D4A6);
+ Entity("lAarr", 0x21DA);
+ Entity("lacute", 0x013A);
+ Entity("Lacute", 0x0139);
+ Entity("laemptyv", 0x29B4);
+ Entity("lagran", 0x2112);
+ Entity("lambda", 0x03BB);
+ Entity("Lambda", 0x039B);
+ Entity("lang", 0x2329);
+ Entity("Lang", 0x27EA);
+ Entity("langd", 0x2991);
+ Entity("langle", 0x2329);
+ Entity("lap", 0x2A85);
+ Entity("Laplacetrf", 0x2112);
+ Entity("laquo", 0x00AB);
+ Entity("larr", 0x2190);
+ Entity("lArr", 0x21D0);
+ Entity("Larr", 0x219E);
+ Entity("larrb", 0x21E4);
+ Entity("larrbfs", 0x291F);
+ Entity("larrfs", 0x291D);
+ Entity("larrhk", 0x21A9);
+ Entity("larrlp", 0x21AB);
+ Entity("larrpl", 0x2939);
+ Entity("larrsim", 0x2973);
+ Entity("larrtl", 0x21A2);
+ Entity("lat", 0x2AAB);
+ Entity("latail", 0x2919);
+ Entity("lAtail", 0x291B);
+ Entity("late", 0x2AAD);
+ Entity("lbarr", 0x290C);
+ Entity("lBarr", 0x290E);
+ Entity("lbbrk", 0x2997);
+ Entity("lbrace", 0x007B);
+ Entity("lbrack", 0x005B);
+ Entity("lbrke", 0x298B);
+ Entity("lbrksld", 0x298F);
+ Entity("lbrkslu", 0x298D);
+ Entity("lcaron", 0x013E);
+ Entity("Lcaron", 0x013D);
+ Entity("lcedil", 0x013C);
+ Entity("Lcedil", 0x013B);
+ Entity("lceil", 0x2308);
+ Entity("lcub", 0x007B);
+ Entity("lcy", 0x043B);
+ Entity("Lcy", 0x041B);
+ Entity("ldca", 0x2936);
+ Entity("ldquo", 0x201C);
+ Entity("ldquor", 0x201E);
+ Entity("ldrdhar", 0x2967);
+ Entity("ldrushar", 0x294B);
+ Entity("ldsh", 0x21B2);
+ Entity("le", 0x2264);
+ Entity("lE", 0x2266);
+ Entity("LeftAngleBracket", 0x2329);
+ Entity("leftarrow", 0x2190);
+ Entity("Leftarrow", 0x21D0);
+ Entity("LeftArrowBar", 0x21E4);
+ Entity("LeftArrowRightArrow", 0x21C6);
+ Entity("leftarrowtail", 0x21A2);
+ Entity("LeftCeiling", 0x2308);
+ Entity("LeftDoubleBracket", 0x27E6);
+ Entity("LeftDownTeeVector", 0x2961);
+ Entity("LeftDownVector", 0x21C3);
+ Entity("LeftDownVectorBar", 0x2959);
+ Entity("LeftFloor", 0x230A);
+ Entity("leftharpoondown", 0x21BD);
+ Entity("leftharpoonup", 0x21BC);
+ Entity("leftleftarrows", 0x21C7);
+ Entity("leftrightarrow", 0x2194);
+ Entity("Leftrightarrow", 0x21D4);
+ Entity("leftrightarrows", 0x21C6);
+ Entity("leftrightharpoons", 0x21CB);
+ Entity("leftrightsquigarrow", 0x21AD);
+ Entity("LeftRightVector", 0x294E);
+ Entity("LeftTee", 0x22A3);
+ Entity("LeftTeeArrow", 0x21A4);
+ Entity("LeftTeeVector", 0x295A);
+ Entity("leftthreetimes", 0x22CB);
+ Entity("LeftTriangle", 0x22B2);
+ Entity("LeftTriangleBar", 0x29CF);
+ Entity("LeftTriangleEqual", 0x22B4);
+ Entity("LeftUpDownVector", 0x2951);
+ Entity("LeftUpTeeVector", 0x2960);
+ Entity("LeftUpVector", 0x21BF);
+ Entity("LeftUpVectorBar", 0x2958);
+ Entity("LeftVector", 0x21BC);
+ Entity("LeftVectorBar", 0x2952);
+ Entity("leg", 0x22DA);
+ Entity("lEg", 0x2A8B);
+ Entity("leq", 0x2264);
+ Entity("leqq", 0x2266);
+ Entity("leqslant", 0x2A7D);
+ Entity("les", 0x2A7D);
+ Entity("lescc", 0x2AA8);
+ Entity("lesdot", 0x2A7F);
+ Entity("lesdoto", 0x2A81);
+ Entity("lesdotor", 0x2A83);
+ Entity("lesges", 0x2A93);
+ Entity("lessapprox", 0x2A85);
+ Entity("lessdot", 0x22D6);
+ Entity("lesseqgtr", 0x22DA);
+ Entity("lesseqqgtr", 0x2A8B);
+ Entity("LessEqualGreater", 0x22DA);
+ Entity("LessFullEqual", 0x2266);
+ Entity("LessGreater", 0x2276);
+ Entity("lessgtr", 0x2276);
+ Entity("LessLess", 0x2AA1);
+ Entity("lesssim", 0x2272);
+ Entity("LessSlantEqual", 0x2A7D);
+ Entity("LessTilde", 0x2272);
+ Entity("lfisht", 0x297C);
+ Entity("lfloor", 0x230A);
+ Entity("lfr", 0x1D529);
+ Entity("Lfr", 0x1D50F);
+ Entity("lg", 0x2276);
+ Entity("lgE", 0x2A91);
+ Entity("lgr", 0x03BB);
+ Entity("Lgr", 0x039B);
+ Entity("lHar", 0x2962);
+ Entity("lhard", 0x21BD);
+ Entity("lharu", 0x21BC);
+ Entity("lharul", 0x296A);
+ Entity("lhblk", 0x2584);
+ Entity("ljcy", 0x0459);
+ Entity("LJcy", 0x0409);
+ Entity("ll", 0x226A);
+ Entity("Ll", 0x22D8);
+ Entity("llarr", 0x21C7);
+ Entity("llcorner", 0x231E);
+ Entity("Lleftarrow", 0x21DA);
+ Entity("llhard", 0x296B);
+ Entity("lltri", 0x25FA);
+ Entity("lmidot", 0x0140);
+ Entity("Lmidot", 0x013F);
+ Entity("lmoust", 0x23B0);
+ Entity("lmoustache", 0x23B0);
+ Entity("lnap", 0x2A89);
+ Entity("lnapprox", 0x2A89);
+ Entity("lne", 0x2A87);
+ Entity("lnE", 0x2268);
+ Entity("lneq", 0x2A87);
+ Entity("lneqq", 0x2268);
+ Entity("lnsim", 0x22E6);
+ Entity("loang", 0x27EC);
+ Entity("loarr", 0x21FD);
+ Entity("lobrk", 0x27E6);
+ Entity("longleftarrow", 0x27F5);
+ Entity("Longleftarrow", 0x27F8);
+ Entity("longleftrightarrow", 0x27F7);
+ Entity("Longleftrightarrow", 0x27FA);
+ Entity("longmapsto", 0x27FC);
+ Entity("longrightarrow", 0x27F6);
+ Entity("Longrightarrow", 0x27F9);
+ Entity("looparrowleft", 0x21AB);
+ Entity("looparrowright", 0x21AC);
+ Entity("lopar", 0x2985);
+ Entity("lopf", 0x1D55D);
+ Entity("Lopf", 0x1D543);
+ Entity("loplus", 0x2A2D);
+ Entity("lotimes", 0x2A34);
+ Entity("lowast", 0x2217);
+ Entity("lowbar", 0x005F);
+ Entity("LowerLeftArrow", 0x2199);
+ Entity("LowerRightArrow", 0x2198);
+ Entity("loz", 0x25CA);
+ Entity("lozenge", 0x25CA);
+ Entity("lozf", 0x29EB);
+ Entity("lpar", 0x0028);
+ Entity("lparlt", 0x2993);
+ Entity("lrarr", 0x21C6);
+ Entity("lrcorner", 0x231F);
+ Entity("lrhar", 0x21CB);
+ Entity("lrhard", 0x296D);
+ Entity("lrm", 0x200E);
+ Entity("lrtri", 0x22BF);
+ Entity("lsaquo", 0x2039);
+ Entity("lscr", 0x1D4C1);
+ Entity("Lscr", 0x2112);
+ Entity("lsh", 0x21B0);
+ Entity("lsim", 0x2272);
+ Entity("lsime", 0x2A8D);
+ Entity("lsimg", 0x2A8F);
+ Entity("lsqb", 0x005B);
+ Entity("lsquo", 0x2018);
+ Entity("lsquor", 0x201A);
+ Entity("lstrok", 0x0142);
+ Entity("Lstrok", 0x0141);
+ Entity("lt", 0x003C);
+ Entity("Lt", 0x226A);
+ Entity("ltcc", 0x2AA6);
+ Entity("ltcir", 0x2A79);
+ Entity("ltdot", 0x22D6);
+ Entity("lthree", 0x22CB);
+ Entity("ltimes", 0x22C9);
+ Entity("ltlarr", 0x2976);
+ Entity("ltquest", 0x2A7B);
+ Entity("ltri", 0x25C3);
+ Entity("ltrie", 0x22B4);
+ Entity("ltrif", 0x25C2);
+ Entity("ltrPar", 0x2996);
+ Entity("lurdshar", 0x294A);
+ Entity("luruhar", 0x2966);
+ Entity("macr", 0x00AF);
+ Entity("male", 0x2642);
+ Entity("malt", 0x2720);
+ Entity("maltese", 0x2720);
+ Entity("map", 0x21A6);
+ Entity("Map", 0x2905);
+ Entity("mapsto", 0x21A6);
+ Entity("mapstodown", 0x21A7);
+ Entity("mapstoleft", 0x21A4);
+ Entity("mapstoup", 0x21A5);
+ Entity("marker", 0x25AE);
+ Entity("mcomma", 0x2A29);
+ Entity("mcy", 0x043C);
+ Entity("Mcy", 0x041C);
+ Entity("mdash", 0x2014);
+ Entity("mDDot", 0x223A);
+ Entity("measuredangle", 0x2221);
+ Entity("MediumSpace", 0x205F);
+ Entity("Mellintrf", 0x2133);
+ Entity("mfr", 0x1D52A);
+ Entity("Mfr", 0x1D510);
+ Entity("mgr", 0x03BC);
+ Entity("Mgr", 0x039C);
+ Entity("mho", 0x2127);
+ Entity("micro", 0x00B5);
+ Entity("mid", 0x2223);
+ Entity("midast", 0x002A);
+ Entity("midcir", 0x2AF0);
+ Entity("middot", 0x00B7);
+ Entity("minus", 0x2212);
+ Entity("minusb", 0x229F);
+ Entity("minusd", 0x2238);
+ Entity("minusdu", 0x2A2A);
+ Entity("MinusPlus", 0x2213);
+ Entity("mlcp", 0x2ADB);
+ Entity("mldr", 0x2026);
+ Entity("mnplus", 0x2213);
+ Entity("models", 0x22A7);
+ Entity("mopf", 0x1D55E);
+ Entity("Mopf", 0x1D544);
+ Entity("mp", 0x2213);
+ Entity("mscr", 0x1D4C2);
+ Entity("Mscr", 0x2133);
+ Entity("mstpos", 0x223E);
+ Entity("mu", 0x03BC);
+ Entity("Mu", 0x039C);
+ Entity("multimap", 0x22B8);
+ Entity("mumap", 0x22B8);
+ Entity("nabla", 0x2207);
+ Entity("nacute", 0x0144);
+ Entity("Nacute", 0x0143);
+ Entity("nap", 0x2249);
+ Entity("napos", 0x0149);
+ Entity("napprox", 0x2249);
+ Entity("natur", 0x266E);
+ Entity("natural", 0x266E);
+ Entity("naturals", 0x2115);
+ Entity("nbsp", 0x00A0);
+ Entity("ncap", 0x2A43);
+ Entity("ncaron", 0x0148);
+ Entity("Ncaron", 0x0147);
+ Entity("ncedil", 0x0146);
+ Entity("Ncedil", 0x0145);
+ Entity("ncong", 0x2247);
+ Entity("ncup", 0x2A42);
+ Entity("ncy", 0x043D);
+ Entity("Ncy", 0x041D);
+ Entity("ndash", 0x2013);
+ Entity("ne", 0x2260);
+ Entity("nearhk", 0x2924);
+ Entity("nearr", 0x2197);
+ Entity("neArr", 0x21D7);
+ Entity("nearrow", 0x2197);
+ Entity("NegativeMediumSpace", 0x200B);
+ Entity("NegativeThickSpace", 0x200B);
+ Entity("NegativeThinSpace", 0x200B);
+ Entity("NegativeVeryThinSpace", 0x200B);
+ Entity("nequiv", 0x2262);
+ Entity("nesear", 0x2928);
+ Entity("NestedGreaterGreater", 0x226B);
+ Entity("NestedLessLess", 0x226A);
+ Entity("NewLine", 0x000A);
+ Entity("nexist", 0x2204);
+ Entity("nexists", 0x2204);
+ Entity("nfr", 0x1D52B);
+ Entity("Nfr", 0x1D511);
+ Entity("nge", 0x2271);
+ Entity("ngeq", 0x2271);
+ Entity("ngr", 0x03BD);
+ Entity("Ngr", 0x039D);
+ Entity("ngsim", 0x2275);
+ Entity("ngt", 0x226F);
+ Entity("ngtr", 0x226F);
+ Entity("nharr", 0x21AE);
+ Entity("nhArr", 0x21CE);
+ Entity("nhpar", 0x2AF2);
+ Entity("ni", 0x220B);
+ Entity("nis", 0x22FC);
+ Entity("nisd", 0x22FA);
+ Entity("niv", 0x220B);
+ Entity("njcy", 0x045A);
+ Entity("NJcy", 0x040A);
+ Entity("nlarr", 0x219A);
+ Entity("nlArr", 0x21CD);
+ Entity("nldr", 0x2025);
+ Entity("nle", 0x2270);
+ Entity("nleftarrow", 0x219A);
+ Entity("nLeftarrow", 0x21CD);
+ Entity("nleftrightarrow", 0x21AE);
+ Entity("nLeftrightarrow", 0x21CE);
+ Entity("nleq", 0x2270);
+ Entity("nless", 0x226E);
+ Entity("nlsim", 0x2274);
+ Entity("nlt", 0x226E);
+ Entity("nltri", 0x22EA);
+ Entity("nltrie", 0x22EC);
+ Entity("nmid", 0x2224);
+ Entity("NoBreak", 0x2060);
+ Entity("NonBreakingSpace", 0x00A0);
+ Entity("nopf", 0x1D55F);
+ Entity("Nopf", 0x2115);
+ Entity("not", 0x00AC);
+ Entity("Not", 0x2AEC);
+ Entity("NotCongruent", 0x2262);
+ Entity("NotCupCap", 0x226D);
+ Entity("NotDoubleVerticalBar", 0x2226);
+ Entity("NotElement", 0x2209);
+ Entity("NotEqual", 0x2260);
+ Entity("NotExists", 0x2204);
+ Entity("NotGreater", 0x226F);
+ Entity("NotGreaterEqual", 0x2271);
+ Entity("NotGreaterLess", 0x2279);
+ Entity("NotGreaterTilde", 0x2275);
+ Entity("notin", 0x2209);
+ Entity("notinva", 0x2209);
+ Entity("notinvb", 0x22F7);
+ Entity("notinvc", 0x22F6);
+ Entity("NotLeftTriangle", 0x22EA);
+ Entity("NotLeftTriangleEqual", 0x22EC);
+ Entity("NotLess", 0x226E);
+ Entity("NotLessEqual", 0x2270);
+ Entity("NotLessGreater", 0x2278);
+ Entity("NotLessTilde", 0x2274);
+ Entity("notni", 0x220C);
+ Entity("notniva", 0x220C);
+ Entity("notnivb", 0x22FE);
+ Entity("notnivc", 0x22FD);
+ Entity("NotPrecedes", 0x2280);
+ Entity("NotPrecedesSlantEqual", 0x22E0);
+ Entity("NotReverseElement", 0x220C);
+ Entity("NotRightTriangle", 0x22EB);
+ Entity("NotRightTriangleEqual", 0x22ED);
+ Entity("NotSquareSubsetEqual", 0x22E2);
+ Entity("NotSquareSupersetEqual", 0x22E3);
+ Entity("NotSubsetEqual", 0x2288);
+ Entity("NotSucceeds", 0x2281);
+ Entity("NotSucceedsSlantEqual", 0x22E1);
+ Entity("NotSupersetEqual", 0x2289);
+ Entity("NotTilde", 0x2241);
+ Entity("NotTildeEqual", 0x2244);
+ Entity("NotTildeFullEqual", 0x2247);
+ Entity("NotTildeTilde", 0x2249);
+ Entity("NotVerticalBar", 0x2224);
+ Entity("npar", 0x2226);
+ Entity("nparallel", 0x2226);
+ Entity("npolint", 0x2A14);
+ Entity("npr", 0x2280);
+ Entity("nprcue", 0x22E0);
+ Entity("nprec", 0x2280);
+ Entity("nrarr", 0x219B);
+ Entity("nrArr", 0x21CF);
+ Entity("nrightarrow", 0x219B);
+ Entity("nRightarrow", 0x21CF);
+ Entity("nrtri", 0x22EB);
+ Entity("nrtrie", 0x22ED);
+ Entity("nsc", 0x2281);
+ Entity("nsccue", 0x22E1);
+ Entity("nscr", 0x1D4C3);
+ Entity("Nscr", 0x1D4A9);
+ Entity("nshortmid", 0x2224);
+ Entity("nshortparallel", 0x2226);
+ Entity("nsim", 0x2241);
+ Entity("nsime", 0x2244);
+ Entity("nsimeq", 0x2244);
+ Entity("nsmid", 0x2224);
+ Entity("nspar", 0x2226);
+ Entity("nsqsube", 0x22E2);
+ Entity("nsqsupe", 0x22E3);
+ Entity("nsub", 0x2284);
+ Entity("nsube", 0x2288);
+ Entity("nsubseteq", 0x2288);
+ Entity("nsucc", 0x2281);
+ Entity("nsup", 0x2285);
+ Entity("nsupe", 0x2289);
+ Entity("nsupseteq", 0x2289);
+ Entity("ntgl", 0x2279);
+ Entity("ntilde", 0x00F1);
+ Entity("Ntilde", 0x00D1);
+ Entity("ntlg", 0x2278);
+ Entity("ntriangleleft", 0x22EA);
+ Entity("ntrianglelefteq", 0x22EC);
+ Entity("ntriangleright", 0x22EB);
+ Entity("ntrianglerighteq", 0x22ED);
+ Entity("nu", 0x03BD);
+ Entity("Nu", 0x039D);
+ Entity("num", 0x0023);
+ Entity("numero", 0x2116);
+ Entity("numsp", 0x2007);
+ Entity("nvdash", 0x22AC);
+ Entity("nvDash", 0x22AD);
+ Entity("nVdash", 0x22AE);
+ Entity("nVDash", 0x22AF);
+ Entity("nvHarr", 0x2904);
+ Entity("nvinfin", 0x29DE);
+ Entity("nvlArr", 0x2902);
+ Entity("nvrArr", 0x2903);
+ Entity("nwarhk", 0x2923);
+ Entity("nwarr", 0x2196);
+ Entity("nwArr", 0x21D6);
+ Entity("nwarrow", 0x2196);
+ Entity("nwnear", 0x2927);
+ Entity("oacgr", 0x03CC);
+ Entity("Oacgr", 0x038C);
+ Entity("oacute", 0x00F3);
+ Entity("Oacute", 0x00D3);
+ Entity("oast", 0x229B);
+ Entity("ocir", 0x229A);
+ Entity("ocirc", 0x00F4);
+ Entity("Ocirc", 0x00D4);
+ Entity("ocy", 0x043E);
+ Entity("Ocy", 0x041E);
+ Entity("odash", 0x229D);
+ Entity("odblac", 0x0151);
+ Entity("Odblac", 0x0150);
+ Entity("odiv", 0x2A38);
+ Entity("odot", 0x2299);
+ Entity("odsold", 0x29BC);
+ Entity("oelig", 0x0153);
+ Entity("OElig", 0x0152);
+ Entity("ofcir", 0x29BF);
+ Entity("ofr", 0x1D52C);
+ Entity("Ofr", 0x1D512);
+ Entity("ogon", 0x02DB);
+ Entity("ogr", 0x03BF);
+ Entity("Ogr", 0x039F);
+ Entity("ograve", 0x00F2);
+ Entity("Ograve", 0x00D2);
+ Entity("ogt", 0x29C1);
+ Entity("ohacgr", 0x03CE);
+ Entity("OHacgr", 0x038F);
+ Entity("ohbar", 0x29B5);
+ Entity("ohgr", 0x03C9);
+ Entity("OHgr", 0x03A9);
+ Entity("ohm", 0x2126);
+ Entity("oint", 0x222E);
+ Entity("olarr", 0x21BA);
+ Entity("olcir", 0x29BE);
+ Entity("olcross", 0x29BB);
+ Entity("oline", 0x203E);
+ Entity("olt", 0x29C0);
+ Entity("omacr", 0x014D);
+ Entity("Omacr", 0x014C);
+ Entity("omega", 0x03C9);
+ Entity("Omega", 0x03A9);
+ Entity("omicron", 0x03BF);
+ Entity("Omicron", 0x039F);
+ Entity("omid", 0x29B6);
+ Entity("ominus", 0x2296);
+ Entity("oopf", 0x1D560);
+ Entity("Oopf", 0x1D546);
+ Entity("opar", 0x29B7);
+ Entity("OpenCurlyDoubleQuote", 0x201C);
+ Entity("OpenCurlyQuote", 0x2018);
+ Entity("operp", 0x29B9);
+ Entity("oplus", 0x2295);
+ Entity("or", 0x2228);
+ Entity("Or", 0x2A54);
+ Entity("orarr", 0x21BB);
+ Entity("ord", 0x2A5D);
+ Entity("order", 0x2134);
+ Entity("orderof", 0x2134);
+ Entity("ordf", 0x00AA);
+ Entity("ordm", 0x00BA);
+ Entity("origof", 0x22B6);
+ Entity("oror", 0x2A56);
+ Entity("orslope", 0x2A57);
+ Entity("orv", 0x2A5B);
+ Entity("oS", 0x24C8);
+ Entity("oscr", 0x2134);
+ Entity("Oscr", 0x1D4AA);
+ Entity("oslash", 0x00F8);
+ Entity("Oslash", 0x00D8);
+ Entity("osol", 0x2298);
+ Entity("otilde", 0x00F5);
+ Entity("Otilde", 0x00D5);
+ Entity("otimes", 0x2297);
+ Entity("Otimes", 0x2A37);
+ Entity("otimesas", 0x2A36);
+ Entity("ouml", 0x00F6);
+ Entity("Ouml", 0x00D6);
+ Entity("ovbar", 0x233D);
+ Entity("OverBar", 0x00AF);
+ Entity("OverBrace", 0xFE37);
+ Entity("OverBracket", 0x23B4);
+ Entity("OverParenthesis", 0xFE35);
+ Entity("par", 0x2225);
+ Entity("para", 0x00B6);
+ Entity("parallel", 0x2225);
+ Entity("parsim", 0x2AF3);
+ Entity("parsl", 0x2AFD);
+ Entity("part", 0x2202);
+ Entity("PartialD", 0x2202);
+ Entity("pcy", 0x043F);
+ Entity("Pcy", 0x041F);
+ Entity("percnt", 0x0025);
+ Entity("period", 0x002E);
+ Entity("permil", 0x2030);
+ Entity("perp", 0x22A5);
+ Entity("pertenk", 0x2031);
+ Entity("pfr", 0x1D52D);
+ Entity("Pfr", 0x1D513);
+ Entity("pgr", 0x03C0);
+ Entity("Pgr", 0x03A0);
+ Entity("phgr", 0x03C6);
+ Entity("PHgr", 0x03A6);
+ Entity("phi", 0x03D5);
+ Entity("Phi", 0x03A6);
+ Entity("phiv", 0x03C6);
+ Entity("phmmat", 0x2133);
+ Entity("phone", 0x260E);
+ Entity("pi", 0x03C0);
+ Entity("Pi", 0x03A0);
+ Entity("pitchfork", 0x22D4);
+ Entity("piv", 0x03D6);
+ Entity("planck", 0x210F);
+ Entity("planckh", 0x210E);
+ Entity("plankv", 0x210F);
+ Entity("plus", 0x002B);
+ Entity("plusacir", 0x2A23);
+ Entity("plusb", 0x229E);
+ Entity("pluscir", 0x2A22);
+ Entity("plusdo", 0x2214);
+ Entity("plusdu", 0x2A25);
+ Entity("pluse", 0x2A72);
+ Entity("PlusMinus", 0x00B1);
+ Entity("plusmn", 0x00B1);
+ Entity("plussim", 0x2A26);
+ Entity("plustwo", 0x2A27);
+ Entity("pm", 0x00B1);
+ Entity("Poincareplane", 0x210C);
+ Entity("pointint", 0x2A15);
+ Entity("popf", 0x1D561);
+ Entity("Popf", 0x2119);
+ Entity("pound", 0x00A3);
+ Entity("pr", 0x227A);
+ Entity("Pr", 0x2ABB);
+ Entity("prap", 0x2AB7);
+ Entity("prcue", 0x227C);
+ Entity("pre", 0x2AAF);
+ Entity("prE", 0x2AB3);
+ Entity("prec", 0x227A);
+ Entity("precapprox", 0x2AB7);
+ Entity("preccurlyeq", 0x227C);
+ Entity("Precedes", 0x227A);
+ Entity("PrecedesEqual", 0x2AAF);
+ Entity("PrecedesSlantEqual", 0x227C);
+ Entity("PrecedesTilde", 0x227E);
+ Entity("preceq", 0x2AAF);
+ Entity("precnapprox", 0x2AB9);
+ Entity("precneqq", 0x2AB5);
+ Entity("precnsim", 0x22E8);
+ Entity("precsim", 0x227E);
+ Entity("prime", 0x2032);
+ Entity("Prime", 0x2033);
+ Entity("primes", 0x2119);
+ Entity("prnap", 0x2AB9);
+ Entity("prnE", 0x2AB5);
+ Entity("prnsim", 0x22E8);
+ Entity("prod", 0x220F);
+ Entity("Product", 0x220F);
+ Entity("profalar", 0x232E);
+ Entity("profline", 0x2312);
+ Entity("profsurf", 0x2313);
+ Entity("prop", 0x221D);
+ Entity("Proportion", 0x2237);
+ Entity("Proportional", 0x221D);
+ Entity("propto", 0x221D);
+ Entity("prsim", 0x227E);
+ Entity("prurel", 0x22B0);
+ Entity("pscr", 0x1D4C5);
+ Entity("Pscr", 0x1D4AB);
+ Entity("psgr", 0x03C8);
+ Entity("PSgr", 0x03A8);
+ Entity("psi", 0x03C8);
+ Entity("Psi", 0x03A8);
+ Entity("puncsp", 0x2008);
+ Entity("qfr", 0x1D52E);
+ Entity("Qfr", 0x1D514);
+ Entity("qint", 0x2A0C);
+ Entity("qopf", 0x1D562);
+ Entity("Qopf", 0x211A);
+ Entity("qprime", 0x2057);
+ Entity("qscr", 0x1D4C6);
+ Entity("Qscr", 0x1D4AC);
+ Entity("quaternions", 0x210D);
+ Entity("quatint", 0x2A16);
+ Entity("quest", 0x003F);
+ Entity("questeq", 0x225F);
+ Entity("quot", 0x0022);
+ Entity("rAarr", 0x21DB);
+ Entity("race", 0x29DA);
+ Entity("racute", 0x0155);
+ Entity("Racute", 0x0154);
+ Entity("radic", 0x221A);
+ Entity("raemptyv", 0x29B3);
+ Entity("rang", 0x232A);
+ Entity("Rang", 0x27EB);
+ Entity("rangd", 0x2992);
+ Entity("range", 0x29A5);
+ Entity("rangle", 0x232A);
+ Entity("raquo", 0x00BB);
+ Entity("rarr", 0x2192);
+ Entity("rArr", 0x21D2);
+ Entity("Rarr", 0x21A0);
+ Entity("rarrap", 0x2975);
+ Entity("rarrb", 0x21E5);
+ Entity("rarrbfs", 0x2920);
+ Entity("rarrc", 0x2933);
+ Entity("rarrfs", 0x291E);
+ Entity("rarrhk", 0x21AA);
+ Entity("rarrlp", 0x21AC);
+ Entity("rarrpl", 0x2945);
+ Entity("rarrsim", 0x2974);
+ Entity("rarrtl", 0x21A3);
+ Entity("Rarrtl", 0x2916);
+ Entity("rarrw", 0x219D);
+ Entity("ratail", 0x291A);
+ Entity("rAtail", 0x291C);
+ Entity("ratio", 0x2236);
+ Entity("rationals", 0x211A);
+ Entity("rbarr", 0x290D);
+ Entity("rBarr", 0x290F);
+ Entity("RBarr", 0x2910);
+ Entity("rbbrk", 0x2998);
+ Entity("rbrace", 0x007D);
+ Entity("rbrack", 0x005D);
+ Entity("rbrke", 0x298C);
+ Entity("rbrksld", 0x298E);
+ Entity("rbrkslu", 0x2990);
+ Entity("rcaron", 0x0159);
+ Entity("Rcaron", 0x0158);
+ Entity("rcedil", 0x0157);
+ Entity("Rcedil", 0x0156);
+ Entity("rceil", 0x2309);
+ Entity("rcub", 0x007D);
+ Entity("rcy", 0x0440);
+ Entity("Rcy", 0x0420);
+ Entity("rdca", 0x2937);
+ Entity("rdldhar", 0x2969);
+ Entity("rdquo", 0x201D);
+ Entity("rdquor", 0x201D);
+ Entity("rdsh", 0x21B3);
+ Entity("Re", 0x211C);
+ Entity("real", 0x211C);
+ Entity("realine", 0x211B);
+ Entity("realpart", 0x211C);
+ Entity("reals", 0x211D);
+ Entity("rect", 0x25AD);
+ Entity("reg", 0x00AE);
+ Entity("ReverseElement", 0x220B);
+ Entity("ReverseEquilibrium", 0x21CB);
+ Entity("ReverseUpEquilibrium", 0x296F);
+ Entity("rfisht", 0x297D);
+ Entity("rfloor", 0x230B);
+ Entity("rfr", 0x1D52F);
+ Entity("Rfr", 0x211C);
+ Entity("rgr", 0x03C1);
+ Entity("Rgr", 0x03A1);
+ Entity("rHar", 0x2964);
+ Entity("rhard", 0x21C1);
+ Entity("rharu", 0x21C0);
+ Entity("rharul", 0x296C);
+ Entity("rho", 0x03C1);
+ Entity("Rho", 0x03A1);
+ Entity("rhov", 0x03F1);
+ Entity("RightAngleBracket", 0x232A);
+ Entity("rightarrow", 0x2192);
+ Entity("Rightarrow", 0x21D2);
+ Entity("RightArrowBar", 0x21E5);
+ Entity("RightArrowLeftArrow", 0x21C4);
+ Entity("rightarrowtail", 0x21A3);
+ Entity("RightCeiling", 0x2309);
+ Entity("RightDoubleBracket", 0x27E7);
+ Entity("RightDownTeeVector", 0x295D);
+ Entity("RightDownVector", 0x21C2);
+ Entity("RightDownVectorBar", 0x2955);
+ Entity("RightFloor", 0x230B);
+ Entity("rightharpoondown", 0x21C1);
+ Entity("rightharpoonup", 0x21C0);
+ Entity("rightleftarrows", 0x21C4);
+ Entity("rightleftharpoons", 0x21CC);
+ Entity("rightrightarrows", 0x21C9);
+ Entity("rightsquigarrow", 0x219D);
+ Entity("RightTee", 0x22A2);
+ Entity("RightTeeArrow", 0x21A6);
+ Entity("RightTeeVector", 0x295B);
+ Entity("rightthreetimes", 0x22CC);
+ Entity("RightTriangle", 0x22B3);
+ Entity("RightTriangleBar", 0x29D0);
+ Entity("RightTriangleEqual", 0x22B5);
+ Entity("RightUpDownVector", 0x294F);
+ Entity("RightUpTeeVector", 0x295C);
+ Entity("RightUpVector", 0x21BE);
+ Entity("RightUpVectorBar", 0x2954);
+ Entity("RightVector", 0x21C0);
+ Entity("RightVectorBar", 0x2953);
+ Entity("ring", 0x02DA);
+ Entity("risingdotseq", 0x2253);
+ Entity("rlarr", 0x21C4);
+ Entity("rlhar", 0x21CC);
+ Entity("rlm", 0x200F);
+ Entity("rmoust", 0x23B1);
+ Entity("rmoustache", 0x23B1);
+ Entity("rnmid", 0x2AEE);
+ Entity("roang", 0x27ED);
+ Entity("roarr", 0x21FE);
+ Entity("robrk", 0x27E7);
+ Entity("ropar", 0x2986);
+ Entity("ropf", 0x1D563);
+ Entity("Ropf", 0x211D);
+ Entity("roplus", 0x2A2E);
+ Entity("rotimes", 0x2A35);
+ Entity("RoundImplies", 0x2970);
+ Entity("rpar", 0x0029);
+ Entity("rpargt", 0x2994);
+ Entity("rppolint", 0x2A12);
+ Entity("rrarr", 0x21C9);
+ Entity("Rrightarrow", 0x21DB);
+ Entity("rsaquo", 0x203A);
+ Entity("rscr", 0x1D4C7);
+ Entity("Rscr", 0x211B);
+ Entity("rsh", 0x21B1);
+ Entity("rsqb", 0x005D);
+ Entity("rsquo", 0x2019);
+ Entity("rsquor", 0x2019);
+ Entity("rthree", 0x22CC);
+ Entity("rtimes", 0x22CA);
+ Entity("rtri", 0x25B9);
+ Entity("rtrie", 0x22B5);
+ Entity("rtrif", 0x25B8);
+ Entity("rtriltri", 0x29CE);
+ Entity("RuleDelayed", 0x29F4);
+ Entity("ruluhar", 0x2968);
+ Entity("rx", 0x211E);
+ Entity("sacute", 0x015B);
+ Entity("Sacute", 0x015A);
+ Entity("sbquo", 0x201A);
+ Entity("sc", 0x227B);
+ Entity("Sc", 0x2ABC);
+ Entity("scap", 0x2AB8);
+ Entity("scaron", 0x0161);
+ Entity("Scaron", 0x0160);
+ Entity("sccue", 0x227D);
+ Entity("sce", 0x2AB0);
+ Entity("scE", 0x2AB4);
+ Entity("scedil", 0x015F);
+ Entity("Scedil", 0x015E);
+ Entity("scirc", 0x015D);
+ Entity("Scirc", 0x015C);
+ Entity("scnap", 0x2ABA);
+ Entity("scnE", 0x2AB6);
+ Entity("scnsim", 0x22E9);
+ Entity("scpolint", 0x2A13);
+ Entity("scsim", 0x227F);
+ Entity("scy", 0x0441);
+ Entity("Scy", 0x0421);
+ Entity("sdot", 0x22C5);
+ Entity("sdotb", 0x22A1);
+ Entity("sdote", 0x2A66);
+ Entity("searhk", 0x2925);
+ Entity("searr", 0x2198);
+ Entity("seArr", 0x21D8);
+ Entity("searrow", 0x2198);
+ Entity("sect", 0x00A7);
+ Entity("semi", 0x003B);
+ Entity("seswar", 0x2929);
+ Entity("setminus", 0x2216);
+ Entity("setmn", 0x2216);
+ Entity("sext", 0x2736);
+ Entity("sfgr", 0x03C2);
+ Entity("sfr", 0x1D530);
+ Entity("Sfr", 0x1D516);
+ Entity("sfrown", 0x2322);
+ Entity("sgr", 0x03C3);
+ Entity("Sgr", 0x03A3);
+ Entity("sharp", 0x266F);
+ Entity("shchcy", 0x0449);
+ Entity("SHCHcy", 0x0429);
+ Entity("shcy", 0x0448);
+ Entity("SHcy", 0x0428);
+ Entity("ShortDownArrow", 0x2193);
+ Entity("ShortLeftArrow", 0x2190);
+ Entity("shortmid", 0x2223);
+ Entity("shortparallel", 0x2225);
+ Entity("ShortRightArrow", 0x2192);
+ Entity("ShortUpArrow", 0x2191);
+ Entity("shy", 0x00AD);
+ Entity("sigma", 0x03C3);
+ Entity("Sigma", 0x03A3);
+ Entity("sigmaf", 0x03C2);
+ Entity("sigmav", 0x03C2);
+ Entity("sim", 0x223C);
+ Entity("simdot", 0x2A6A);
+ Entity("sime", 0x2243);
+ Entity("simeq", 0x2243);
+ Entity("simg", 0x2A9E);
+ Entity("simgE", 0x2AA0);
+ Entity("siml", 0x2A9D);
+ Entity("simlE", 0x2A9F);
+ Entity("simne", 0x2246);
+ Entity("simplus", 0x2A24);
+ Entity("simrarr", 0x2972);
+ Entity("slarr", 0x2190);
+ Entity("SmallCircle", 0x2218);
+ Entity("smallsetminus", 0x2216);
+ Entity("smashp", 0x2A33);
+ Entity("smeparsl", 0x29E4);
+ Entity("smid", 0x2223);
+ Entity("smile", 0x2323);
+ Entity("smt", 0x2AAA);
+ Entity("smte", 0x2AAC);
+ Entity("softcy", 0x044C);
+ Entity("SOFTcy", 0x042C);
+ Entity("sol", 0x002F);
+ Entity("solb", 0x29C4);
+ Entity("solbar", 0x233F);
+ Entity("sopf", 0x1D564);
+ Entity("Sopf", 0x1D54A);
+ Entity("spades", 0x2660);
+ Entity("spadesuit", 0x2660);
+ Entity("spar", 0x2225);
+ Entity("sqcap", 0x2293);
+ Entity("sqcup", 0x2294);
+ Entity("Sqrt", 0x221A);
+ Entity("sqsub", 0x228F);
+ Entity("sqsube", 0x2291);
+ Entity("sqsubset", 0x228F);
+ Entity("sqsubseteq", 0x2291);
+ Entity("sqsup", 0x2290);
+ Entity("sqsupe", 0x2292);
+ Entity("sqsupset", 0x2290);
+ Entity("sqsupseteq", 0x2292);
+ Entity("squ", 0x25A1);
+ Entity("square", 0x25A1);
+ Entity("SquareIntersection", 0x2293);
+ Entity("SquareSubset", 0x228F);
+ Entity("SquareSubsetEqual", 0x2291);
+ Entity("SquareSuperset", 0x2290);
+ Entity("SquareSupersetEqual", 0x2292);
+ Entity("SquareUnion", 0x2294);
+ Entity("squarf", 0x25AA);
+ Entity("squf", 0x25AA);
+ Entity("srarr", 0x2192);
+ Entity("sscr", 0x1D4C8);
+ Entity("Sscr", 0x1D4AE);
+ Entity("ssetmn", 0x2216);
+ Entity("ssmile", 0x2323);
+ Entity("sstarf", 0x22C6);
+ Entity("star", 0x2606);
+ Entity("Star", 0x22C6);
+ Entity("starf", 0x2605);
+ Entity("straightepsilon", 0x03F5);
+ Entity("straightphi", 0x03D5);
+ Entity("strns", 0x00AF);
+ Entity("sub", 0x2282);
+ Entity("Sub", 0x22D0);
+ Entity("subdot", 0x2ABD);
+ Entity("sube", 0x2286);
+ Entity("subE", 0x2AC5);
+ Entity("subedot", 0x2AC3);
+ Entity("submult", 0x2AC1);
+ Entity("subne", 0x228A);
+ Entity("subnE", 0x2ACB);
+ Entity("subplus", 0x2ABF);
+ Entity("subrarr", 0x2979);
+ Entity("subset", 0x2282);
+ Entity("Subset", 0x22D0);
+ Entity("subseteq", 0x2286);
+ Entity("subseteqq", 0x2AC5);
+ Entity("SubsetEqual", 0x2286);
+ Entity("subsetneq", 0x228A);
+ Entity("subsetneqq", 0x2ACB);
+ Entity("subsim", 0x2AC7);
+ Entity("subsub", 0x2AD5);
+ Entity("subsup", 0x2AD3);
+ Entity("succ", 0x227B);
+ Entity("succapprox", 0x2AB8);
+ Entity("succcurlyeq", 0x227D);
+ Entity("Succeeds", 0x227B);
+ Entity("SucceedsEqual", 0x2AB0);
+ Entity("SucceedsSlantEqual", 0x227D);
+ Entity("SucceedsTilde", 0x227F);
+ Entity("succeq", 0x2AB0);
+ Entity("succnapprox", 0x2ABA);
+ Entity("succneqq", 0x2AB6);
+ Entity("succnsim", 0x22E9);
+ Entity("succsim", 0x227F);
+ Entity("SuchThat", 0x220B);
+ Entity("sum", 0x2211);
+ Entity("sung", 0x266A);
+ Entity("sup", 0x2283);
+ Entity("Sup", 0x22D1);
+ Entity("sup1", 0x00B9);
+ Entity("sup2", 0x00B2);
+ Entity("sup3", 0x00B3);
+ Entity("supdot", 0x2ABE);
+ Entity("supdsub", 0x2AD8);
+ Entity("supe", 0x2287);
+ Entity("supE", 0x2AC6);
+ Entity("supedot", 0x2AC4);
+ Entity("Superset", 0x2283);
+ Entity("SupersetEqual", 0x2287);
+ Entity("suphsub", 0x2AD7);
+ Entity("suplarr", 0x297B);
+ Entity("supmult", 0x2AC2);
+ Entity("supne", 0x228B);
+ Entity("supnE", 0x2ACC);
+ Entity("supplus", 0x2AC0);
+ Entity("supset", 0x2283);
+ Entity("Supset", 0x22D1);
+ Entity("
<TRUNCATED>
[27/33] lucenenet git commit: Lucene.Net.Analysis (Kuromoji +
SmartCn): Changed System.Text.Encoding.CodePages dependency from
4.4.0-preview1-25305-02 to stable 4.3.0
Posted by ni...@apache.org.
Lucene.Net.Analysis (Kuromoji + SmartCn): Changed System.Text.Encoding.CodePages dependency from 4.4.0-preview1-25305-02 to stable 4.3.0
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/56a18c33
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/56a18c33
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/56a18c33
Branch: refs/heads/master
Commit: 56a18c33e40fadd572f64b4de3d39a6d18a84a8e
Parents: e92525f
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Fri Aug 4 15:25:29 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Fri Aug 4 19:33:35 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Analysis.Kuromoji/project.json | 2 +-
src/Lucene.Net.Analysis.SmartCn/project.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/56a18c33/src/Lucene.Net.Analysis.Kuromoji/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Kuromoji/project.json b/src/Lucene.Net.Analysis.Kuromoji/project.json
index 937b9bf..88a5472 100644
--- a/src/Lucene.Net.Analysis.Kuromoji/project.json
+++ b/src/Lucene.Net.Analysis.Kuromoji/project.json
@@ -47,7 +47,7 @@
"dependencies": {
"NETStandard.Library": "1.6.0",
"System.Globalization.Extensions": "4.3.0",
- "System.Text.Encoding.CodePages": "4.4.0-preview1-25305-02"
+ "System.Text.Encoding.CodePages": "4.3.0"
}
},
"net451": {
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/56a18c33/src/Lucene.Net.Analysis.SmartCn/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.SmartCn/project.json b/src/Lucene.Net.Analysis.SmartCn/project.json
index 0fa0612..0f92980 100644
--- a/src/Lucene.Net.Analysis.SmartCn/project.json
+++ b/src/Lucene.Net.Analysis.SmartCn/project.json
@@ -40,7 +40,7 @@
},
"dependencies": {
"NETStandard.Library": "1.6.0",
- "System.Text.Encoding.CodePages": "4.4.0-preview1-25305-02"
+ "System.Text.Encoding.CodePages": "4.3.0"
}
},
"net451": {
[06/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
new file mode 100644
index 0000000..9be1b5d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
@@ -0,0 +1,662 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support.Threading;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Sequence of parallel or sequential tasks.
+ /// </summary>
+ public class TaskSequence : PerfTask
+ {
+ public static int REPEAT_EXHAUST = -2;
+ private IList<PerfTask> tasks;
+ private int repetitions = 1;
+ private bool parallel;
+ private TaskSequence parent;
+ private bool letChildReport = true;
+ private int rate = 0;
+ private bool perMin = false; // rate, if set, is, by default, be sec.
+ private string seqName;
+ private bool exhausted = false;
+ private bool resetExhausted = false;
+ private PerfTask[] tasksArray;
+ private bool anyExhaustibleTasks;
+ private bool collapsable = false; // to not collapse external sequence named in alg.
+
+ private bool fixedTime; // true if we run for fixed time
+ private double runTimeSec; // how long to run for
+ private readonly long logByTimeMsec;
+
+ public TaskSequence(PerfRunData runData, String name, TaskSequence parent, bool parallel)
+ : base(runData)
+ {
+ collapsable = (name == null);
+ name = (name != null ? name : (parallel ? "Par" : "Seq"));
+ SetName(name);
+ SetSequenceName();
+ this.parent = parent;
+ this.parallel = parallel;
+ tasks = new List<PerfTask>();
+ logByTimeMsec = runData.Config.Get("report.time.step.msec", 0);
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ InitTasksArray();
+ for (int i = 0; i < tasksArray.Length; i++)
+ {
+ tasksArray[i].Dispose();
+ }
+ RunData.DocMaker.Dispose();
+ }
+ }
+
+ private void InitTasksArray()
+ {
+ if (tasksArray == null)
+ {
+ int numTasks = tasks.Count;
+ tasksArray = new PerfTask[numTasks];
+ for (int k = 0; k < numTasks; k++)
+ {
+ tasksArray[k] = tasks[k];
+ anyExhaustibleTasks |= tasksArray[k] is ResetInputsTask;
+ anyExhaustibleTasks |= tasksArray[k] is TaskSequence;
+ }
+ }
+ if (!parallel && logByTimeMsec != 0 && !letChildReport)
+ {
+ countsByTime = new int[1];
+ }
+ }
+
+ /// <summary>
+ /// Gets the parallel.
+ /// </summary>
+ public virtual bool IsParallel
+ {
+ get { return parallel; }
+ }
+
+ /// <summary>
+ /// Gets the repetitions.
+ /// </summary>
+ public virtual int Repetitions
+ {
+ get { return repetitions; }
+ }
+
+ private int[] countsByTime;
+
+ public virtual void SetRunTime(double sec)
+ {
+ runTimeSec = sec;
+ fixedTime = true;
+ }
+
+ /// <summary>
+ /// Sets the repetitions.
+ /// </summary>
+ /// <param name="repetitions">The repetitions to set.</param>
+ public virtual void SetRepetitions(int repetitions)
+ {
+ fixedTime = false;
+ this.repetitions = repetitions;
+ if (repetitions == REPEAT_EXHAUST)
+ {
+ if (IsParallel)
+ {
+ throw new Exception("REPEAT_EXHAUST is not allowed for parallel tasks");
+ }
+ }
+ SetSequenceName();
+ }
+
+ /// <summary>
+ /// Gets the parent.
+ /// </summary>
+ public virtual TaskSequence Parent
+ {
+ get { return parent; }
+ }
+
+ /// <seealso cref="PerfTask.DoLogic()"/>
+ public override int DoLogic()
+ {
+ exhausted = resetExhausted = false;
+ return (parallel ? DoParallelTasks() : DoSerialTasks());
+ }
+
+ private class RunBackgroundTask : ThreadClass
+ {
+ private readonly PerfTask task;
+ private readonly bool letChildReport;
+ private volatile int count;
+
+ public RunBackgroundTask(PerfTask task, bool letChildReport)
+ {
+ this.task = task;
+ this.letChildReport = letChildReport;
+ }
+
+ public virtual void StopNow()
+ {
+ task.StopNow();
+ }
+
+ public virtual int Count
+ {
+ get { return count; }
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ count = task.RunAndMaybeStats(letChildReport);
+ }
+ catch (Exception e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+ }
+
+ private int DoSerialTasks()
+ {
+ if (rate > 0)
+ {
+ return DoSerialTasksWithRate();
+ }
+
+ InitTasksArray();
+ int count = 0;
+
+ long runTime = (long)(runTimeSec * 1000);
+ List<RunBackgroundTask> bgTasks = null;
+
+ long t0 = Support.Time.CurrentTimeMilliseconds();
+ for (int k = 0; fixedTime || (repetitions == REPEAT_EXHAUST && !exhausted) || k < repetitions; k++)
+ {
+ if (Stop)
+ {
+ break;
+ }
+ for (int l = 0; l < tasksArray.Length; l++)
+ {
+ PerfTask task = tasksArray[l];
+ if (task.RunInBackground)
+ {
+ if (bgTasks == null)
+ {
+ bgTasks = new List<RunBackgroundTask>();
+ }
+ RunBackgroundTask bgTask = new RunBackgroundTask(task, letChildReport);
+ bgTask.Priority = (task.BackgroundDeltaPriority + Thread.CurrentThread.Priority);
+ bgTask.Start();
+ bgTasks.Add(bgTask);
+ }
+ else
+ {
+ try
+ {
+ int inc = task.RunAndMaybeStats(letChildReport);
+ count += inc;
+ if (countsByTime != null)
+ {
+ int slot = (int)((Support.Time.CurrentTimeMilliseconds() - t0) / logByTimeMsec);
+ if (slot >= countsByTime.Length)
+ {
+ countsByTime = ArrayUtil.Grow(countsByTime, 1 + slot);
+ }
+ countsByTime[slot] += inc;
+ }
+ if (anyExhaustibleTasks)
+ UpdateExhausted(task);
+ }
+ catch (NoMoreDataException /*e*/)
+ {
+ exhausted = true;
+ }
+ }
+ }
+ if (fixedTime && Support.Time.CurrentTimeMilliseconds() - t0 > runTime)
+ {
+ repetitions = k + 1;
+ break;
+ }
+ }
+
+ if (bgTasks != null)
+ {
+ foreach (RunBackgroundTask bgTask in bgTasks)
+ {
+ bgTask.StopNow();
+ }
+ foreach (RunBackgroundTask bgTask in bgTasks)
+ {
+ bgTask.Join();
+ count += bgTask.Count;
+ }
+ }
+
+ if (countsByTime != null)
+ {
+ RunData.Points.CurrentStats.SetCountsByTime(countsByTime, logByTimeMsec);
+ }
+
+ Stop = false;
+
+ return count;
+ }
+
+ private int DoSerialTasksWithRate()
+ {
+ InitTasksArray();
+ long delayStep = (perMin ? 60000 : 1000) / rate;
+ long nextStartTime = Support.Time.CurrentTimeMilliseconds();
+ int count = 0;
+ long t0 = Support.Time.CurrentTimeMilliseconds();
+ for (int k = 0; (repetitions == REPEAT_EXHAUST && !exhausted) || k < repetitions; k++)
+ {
+ if (Stop)
+ {
+ break;
+ }
+ for (int l = 0; l < tasksArray.Length; l++)
+ {
+ PerfTask task = tasksArray[l];
+ while (!Stop)
+ {
+ long waitMore = nextStartTime - Support.Time.CurrentTimeMilliseconds();
+ if (waitMore > 0)
+ {
+ // TODO: better to use condition to notify
+ Thread.Sleep(1);
+ }
+ else
+ {
+ break;
+ }
+ }
+ if (Stop)
+ {
+ break;
+ }
+ nextStartTime += delayStep; // this aims at avarage rate.
+ try
+ {
+ int inc = task.RunAndMaybeStats(letChildReport);
+ count += inc;
+ if (countsByTime != null)
+ {
+ int slot = (int)((Support.Time.CurrentTimeMilliseconds() - t0) / logByTimeMsec);
+ if (slot >= countsByTime.Length)
+ {
+ countsByTime = ArrayUtil.Grow(countsByTime, 1 + slot);
+ }
+ countsByTime[slot] += inc;
+ }
+
+ if (anyExhaustibleTasks)
+ UpdateExhausted(task);
+ }
+ catch (NoMoreDataException /*e*/)
+ {
+ exhausted = true;
+ }
+ }
+ }
+ Stop = false;
+ return count;
+ }
+
+ // update state regarding exhaustion.
+ private void UpdateExhausted(PerfTask task)
+ {
+ if (task is ResetInputsTask)
+ {
+ exhausted = false;
+ resetExhausted = true;
+ }
+ else if (task is TaskSequence)
+ {
+ TaskSequence t = (TaskSequence)task;
+ if (t.resetExhausted)
+ {
+ exhausted = false;
+ resetExhausted = true;
+ t.resetExhausted = false;
+ }
+ else
+ {
+ exhausted |= t.exhausted;
+ }
+ }
+ }
+
+ private class ParallelTask : ThreadClass
+ {
+ private int count;
+ private readonly PerfTask task;
+ private readonly TaskSequence outerInstance;
+
+ // LUCENENET specific - expose field through property
+ public int Count
+ {
+ get { return count; }
+ }
+
+ // LUCENENET specific - expose field through property
+ public PerfTask Task
+ {
+ get { return task; }
+ }
+
+ public ParallelTask(TaskSequence outerInstance, PerfTask task)
+ {
+ this.outerInstance = outerInstance;
+ this.task = task;
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ int n = task.RunAndMaybeStats(outerInstance.letChildReport);
+ if (outerInstance.anyExhaustibleTasks)
+ {
+ outerInstance.UpdateExhausted(task);
+ }
+ count += n;
+ }
+ catch (NoMoreDataException)
+ {
+ outerInstance.exhausted = true;
+ }
+ catch (Exception e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+ }
+
+ public override void StopNow()
+ {
+ base.StopNow();
+ // Forwards top request to children
+ if (runningParallelTasks != null)
+ {
+ foreach (ParallelTask t in runningParallelTasks)
+ {
+ if (t != null)
+ {
+ t.Task.StopNow();
+ }
+ }
+ }
+ }
+
+ ParallelTask[] runningParallelTasks;
+
+ private int DoParallelTasks()
+ {
+
+ TaskStats stats = RunData.Points.CurrentStats;
+
+ InitTasksArray();
+ ParallelTask[] t = runningParallelTasks = new ParallelTask[repetitions * tasks.Count];
+ // prepare threads
+ int index = 0;
+ for (int k = 0; k < repetitions; k++)
+ {
+ for (int i = 0; i < tasksArray.Length; i++)
+ {
+ PerfTask task = (PerfTask)(tasksArray[i].Clone());
+ t[index++] = new ParallelTask(this, task);
+ }
+ }
+ // run threads
+ StartThreads(t);
+
+ if (Stop)
+ {
+ foreach (ParallelTask task in t)
+ {
+ task.Task.StopNow();
+ }
+ }
+
+ // wait for all threads to complete
+ int count = 0;
+ for (int i = 0; i < t.Length; i++)
+ {
+ t[i].Join();
+ count += t[i].Count;
+ if (t[i].Task is TaskSequence)
+ {
+ TaskSequence sub = (TaskSequence)t[i].Task;
+ if (sub.countsByTime != null)
+ {
+ if (countsByTime == null)
+ {
+ countsByTime = new int[sub.countsByTime.Length];
+ }
+ else if (countsByTime.Length < sub.countsByTime.Length)
+ {
+ countsByTime = ArrayUtil.Grow(countsByTime, sub.countsByTime.Length);
+ }
+ for (int j = 0; j < sub.countsByTime.Length; j++)
+ {
+ countsByTime[j] += sub.countsByTime[j];
+ }
+ }
+ }
+ }
+
+ if (countsByTime != null)
+ {
+ stats.SetCountsByTime(countsByTime, logByTimeMsec);
+ }
+
+ // return total count
+ return count;
+ }
+
+ // run threads
+ private void StartThreads(ParallelTask[] t)
+ {
+ if (rate > 0)
+ {
+ StartlThreadsWithRate(t);
+ return;
+ }
+ for (int i = 0; i < t.Length; i++)
+ {
+ t[i].Start();
+ }
+ }
+
+ // run threads with rate
+ private void StartlThreadsWithRate(ParallelTask[] t)
+ {
+ long delayStep = (perMin ? 60000 : 1000) / rate;
+ long nextStartTime = Support.Time.CurrentTimeMilliseconds();
+ for (int i = 0; i < t.Length; i++)
+ {
+ long waitMore = nextStartTime - Support.Time.CurrentTimeMilliseconds();
+ if (waitMore > 0)
+ {
+ Thread.Sleep((int)waitMore);
+ }
+ nextStartTime += delayStep; // this aims at average rate of starting threads.
+ t[i].Start();
+ }
+ }
+
+ public virtual void AddTask(PerfTask task)
+ {
+ tasks.Add(task);
+ task.Depth = Depth + 1;
+ }
+
+ /// <seealso cref="object.ToString()"/>
+ public override string ToString()
+ {
+ string padd = GetPadding();
+ StringBuilder sb = new StringBuilder(base.ToString());
+ sb.Append(parallel ? " [" : " {");
+ sb.Append(NEW_LINE);
+ foreach (PerfTask task in tasks)
+ {
+ sb.Append(task.ToString());
+ sb.Append(NEW_LINE);
+ }
+ sb.Append(padd);
+ sb.Append(!letChildReport ? ">" : (parallel ? "]" : "}"));
+ if (fixedTime)
+ {
+ sb.AppendFormat(CultureInfo.InvariantCulture, " {0:N}s", runTimeSec);
+ }
+ else if (repetitions > 1)
+ {
+ sb.Append(" * " + repetitions);
+ }
+ else if (repetitions == REPEAT_EXHAUST)
+ {
+ sb.Append(" * EXHAUST");
+ }
+ if (rate > 0)
+ {
+ sb.Append(", rate: " + rate + "/" + (perMin ? "min" : "sec"));
+ }
+ if (RunInBackground)
+ {
+ sb.Append(" &");
+ int x = BackgroundDeltaPriority;
+ if (x != 0)
+ {
+ sb.Append(x);
+ }
+ }
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Execute child tasks in a way that they do not report their time separately.
+ /// </summary>
+ public virtual void SetNoChildReport()
+ {
+ letChildReport = false;
+ foreach (PerfTask task in tasks)
+ {
+ if (task is TaskSequence)
+ {
+ ((TaskSequence)task).SetNoChildReport();
+ }
+ }
+ }
+
+ /// <summary>
+ /// Returns the rate per minute: how many operations should be performed in a minute.
+ /// If 0 this has no effect.
+ /// </summary>
+ /// <returns>The rate per min: how many operations should be performed in a minute.</returns>
+ public virtual int GetRate()
+ {
+ return (perMin ? rate : 60 * rate);
+ }
+
+ /// <summary>
+ ///
+ /// </summary>
+ /// <param name="rate">The rate to set.</param>
+ /// <param name="perMin"></param>
+ public virtual void SetRate(int rate, bool perMin)
+ {
+ this.rate = rate;
+ this.perMin = perMin;
+ SetSequenceName();
+ }
+
+ private void SetSequenceName()
+ {
+ seqName = base.GetName();
+ if (repetitions == REPEAT_EXHAUST)
+ {
+ seqName += "_Exhaust";
+ }
+ else if (repetitions > 1)
+ {
+ seqName += "_" + repetitions;
+ }
+ if (rate > 0)
+ {
+ seqName += "_" + rate + (perMin ? "/min" : "/sec");
+ }
+ if (parallel && seqName.ToLowerInvariant().IndexOf("par") < 0)
+ {
+ seqName += "_Par";
+ }
+ }
+
+ public override string GetName()
+ {
+ return seqName; // override to include more info
+ }
+
+ /// <summary>
+ /// Gets the tasks.
+ /// </summary>
+ public virtual IList<PerfTask> Tasks
+ {
+ get { return tasks; }
+ }
+
+ /// <seealso cref="ICloneable.Clone()"/>
+ public override object Clone()
+ {
+ TaskSequence res = (TaskSequence)base.Clone();
+ res.tasks = new List<PerfTask>();
+ for (int i = 0; i < tasks.Count; i++)
+ {
+ res.tasks.Add((PerfTask)tasks[i].Clone());
+ }
+ return res;
+ }
+
+ /// <summary>
+ /// Return <c>true</c> if can be collapsed in case it is outermost sequence.
+ /// </summary>
+ public virtual bool IsCollapsable
+ {
+ get { return collapsable; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs
new file mode 100644
index 0000000..f69ce2b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs
@@ -0,0 +1,99 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using System;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Update a document, using <see cref="IndexWriter.UpdateDocument(Term, System.Collections.Generic.IEnumerable{IIndexableField})"/>,
+ /// optionally with of a certain size.
+ /// <para/>
+ /// Other side effects: none.
+ /// <para/>
+ /// Takes optional param: document size.
+ /// </summary>
+ public class UpdateDocTask : PerfTask
+ {
+ public UpdateDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ private int docSize = 0;
+
+ // volatile data passed between setup(), doLogic(), tearDown().
+ private Document doc = null;
+
+ public override void Setup()
+ {
+ base.Setup();
+ DocMaker docMaker = RunData.DocMaker;
+ if (docSize > 0)
+ {
+ doc = docMaker.MakeDocument(docSize);
+ }
+ else
+ {
+ doc = docMaker.MakeDocument();
+ }
+ }
+
+ public override void TearDown()
+ {
+ doc = null;
+ base.TearDown();
+ }
+
+ public override int DoLogic()
+ {
+ string docID = doc.Get(DocMaker.ID_FIELD);
+ if (docID == null)
+ {
+ throw new InvalidOperationException("document must define the docid field");
+ }
+ IndexWriter iw = RunData.IndexWriter;
+ iw.UpdateDocument(new Term(DocMaker.ID_FIELD, docID), doc);
+ return 1;
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return "updated " + recsCount + " docs";
+ }
+
+ /// <summary>
+ /// Set the params (docSize only)
+ /// </summary>
+ /// <param name="params">docSize, or 0 for no limit.</param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ docSize = (int)float.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs
new file mode 100644
index 0000000..7dad964
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs
@@ -0,0 +1,36 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Waits for merges to finish.
+ /// </summary>
+ public class WaitForMergesTask : PerfTask
+ {
+ public WaitForMergesTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.IndexWriter.WaitForMerges();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs
new file mode 100644
index 0000000..67f648d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs
@@ -0,0 +1,89 @@
+using System;
+using System.Globalization;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simply waits for the specified (via the parameter) amount
+ /// of time. For example Wait(30s) waits for 30 seconds.
+ /// This is useful with background tasks to control how long
+ /// the tasks run.
+ /// <para/>
+ /// You can specify h, m, or s (hours, minutes, seconds) as
+ /// the trailing time unit. No unit is interpreted as
+ /// seconds.
+ /// </summary>
+ public class WaitTask : PerfTask
+ {
+ private double waitTimeSec;
+
+ public WaitTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ if (@params != null)
+ {
+ int multiplier;
+ if (@params.EndsWith("s", StringComparison.Ordinal))
+ {
+ multiplier = 1;
+ @params = @params.Substring(0, @params.Length - 1);
+ }
+ else if (@params.EndsWith("m", StringComparison.Ordinal))
+ {
+ multiplier = 60;
+ @params = @params.Substring(0, @params.Length - 1);
+ }
+ else if (@params.EndsWith("h", StringComparison.Ordinal))
+ {
+ multiplier = 3600;
+ @params = @params.Substring(0, @params.Length - 1);
+ }
+ else
+ {
+ // Assume seconds
+ multiplier = 1;
+ }
+
+ waitTimeSec = double.Parse(@params, CultureInfo.InvariantCulture) * multiplier;
+ }
+ else
+ {
+ throw new ArgumentException("you must specify the wait time, eg: 10.0s, 4.5m, 2h");
+ }
+ }
+
+ public override int DoLogic()
+ {
+ Thread.Sleep((int)(1000 * waitTimeSec));
+ return 0;
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs
new file mode 100644
index 0000000..3e1f8d8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs
@@ -0,0 +1,64 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Warm reader task: retrieve all reader documents.
+ /// </summary>
+ /// <remarks>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each
+ /// retrieved (non null) document.
+ /// </remarks>
+ public class WarmTask : ReadTask
+ {
+ public WarmTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return false; }
+ }
+
+ public override bool WithSearch
+ {
+ get { return false; }
+ }
+
+ public override bool WithTraverse
+ {
+ get { return false; }
+ }
+
+ public override bool WithWarm
+ {
+ get { return true; }
+ }
+
+ public override IQueryMaker GetQueryMaker()
+ {
+ return null; // not required for this task.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs
new file mode 100644
index 0000000..d70836e
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs
@@ -0,0 +1,72 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="WriteLineDocTask"/> which for Wikipedia input, will write category pages
+ /// to another file, while remaining pages will be written to the original file.
+ /// The categories file is derived from the original file, by adding a prefix "categories-".
+ /// </summary>
+ public class WriteEnwikiLineDocTask : WriteLineDocTask
+ {
+ private readonly TextWriter categoryLineFileOut;
+
+ public WriteEnwikiLineDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ Stream @out = StreamUtils.GetOutputStream(CategoriesLineFile(new FileInfo(m_fname)));
+ categoryLineFileOut = new StreamWriter(@out, Encoding.UTF8);
+ WriteHeader(categoryLineFileOut);
+ }
+
+ /// <summary>Compose categories line file out of original line file</summary>
+ public static FileInfo CategoriesLineFile(FileInfo f)
+ {
+ DirectoryInfo dir = f.Directory;
+ string categoriesName = "categories-" + f.Name;
+ return dir == null ? new FileInfo(categoriesName) : new FileInfo(System.IO.Path.Combine(dir.FullName, categoriesName));
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ categoryLineFileOut.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ protected override TextWriter LineFileOut(Document doc)
+ {
+ IIndexableField titleField = doc.GetField(DocMaker.TITLE_FIELD);
+ if (titleField != null && titleField.GetStringValue().StartsWith("Category:", StringComparison.Ordinal))
+ {
+ return categoryLineFileOut;
+ }
+ return base.LineFileOut(doc);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
new file mode 100644
index 0000000..f9e7546
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
@@ -0,0 +1,238 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A task which writes documents, one line per document. Each line is in the
+ /// following format: title <TAB> date <TAB> body. The output of this
+ /// task can be consumed by <see cref="LineDocSource"/> and is intended
+ /// to save the IO overhead of opening a file per document to be indexed.
+ /// </summary>
+ /// <remarks>
+ /// The format of the output is set according to the output file extension.
+ /// Compression is recommended when the output file is expected to be large.
+ /// See info on file extensions in <see cref="FileType"/>.
+ /// <para/>
+ /// Supports the following parameters:
+ /// <list type="bullet">
+ /// <item><term>line.file.out</term><description>the name of the file to write the output to. That parameter is mandatory. <b>NOTE:</b> the file is re-created.</description></item>
+ /// <item><term>line.fields</term><description>which fields should be written in each line. (optional, default: <see cref="DEFAULT_FIELDS"/>).</description></item>
+ /// <item><term>sufficient.fields</term><description>
+ /// list of field names, separated by comma, which,
+ /// if all of them are missing, the document will be skipped. For example, to require
+ /// that at least one of f1,f2 is not empty, specify: "f1,f2" in this field. To specify
+ /// that no field is required, i.e. that even empty docs should be emitted, specify <b>","</b>
+ /// (optional, default: <see cref="DEFAULT_SUFFICIENT_FIELDS"/>).
+ /// </description></item>
+ /// </list>
+ /// <para/>
+ /// <b>NOTE:</b> this class is not thread-safe and if used by multiple threads the
+ /// output is unspecified (as all will write to the same output file in a
+ /// non-synchronized way).
+ /// </remarks>
+ public class WriteLineDocTask : PerfTask
+ {
+ public static readonly string FIELDS_HEADER_INDICATOR = "FIELDS_HEADER_INDICATOR###";
+
+ public readonly static char SEP = '\t';
+
+ /// <summary>
+ /// Fields to be written by default
+ /// </summary>
+ public static readonly string[] DEFAULT_FIELDS = new string[] {
+ DocMaker.TITLE_FIELD,
+ DocMaker.DATE_FIELD,
+ DocMaker.BODY_FIELD,
+ };
+
+ /// <summary>
+ /// Default fields which at least one of them is required to not skip the doc.
+ /// </summary>
+ public static readonly string DEFAULT_SUFFICIENT_FIELDS = DocMaker.TITLE_FIELD + ',' + DocMaker.BODY_FIELD;
+
+ private int docSize = 0;
+ protected readonly string m_fname;
+ private readonly TextWriter lineFileOut;
+ private readonly DocMaker docMaker;
+ private readonly ThreadLocal<StringBuilder> threadBuffer = new ThreadLocal<StringBuilder>();
+ private readonly ThreadLocal<Regex> threadNormalizer = new ThreadLocal<Regex>();
+ private readonly string[] fieldsToWrite;
+ private readonly bool[] sufficientFields;
+ private readonly bool checkSufficientFields;
+
+
+ public WriteLineDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ Config config = runData.Config;
+ m_fname = config.Get("line.file.out", null);
+ if (m_fname == null)
+ {
+ throw new ArgumentException("line.file.out must be set");
+ }
+ Stream @out = StreamUtils.GetOutputStream(new FileInfo(m_fname));
+ lineFileOut = new StreamWriter(@out, Encoding.UTF8);
+ docMaker = runData.DocMaker;
+
+ // init fields
+ string f2r = config.Get("line.fields", null);
+ if (f2r == null)
+ {
+ fieldsToWrite = DEFAULT_FIELDS;
+ }
+ else
+ {
+ if (f2r.IndexOf(SEP) >= 0)
+ {
+ throw new ArgumentException("line.fields " + f2r + " should not contain the separator char: " + SEP);
+ }
+ fieldsToWrite = f2r.Split(',').TrimEnd();
+ }
+
+ // init sufficient fields
+ sufficientFields = new bool[fieldsToWrite.Length];
+ string suff = config.Get("sufficient.fields", DEFAULT_SUFFICIENT_FIELDS);
+ if (",".Equals(suff))
+ {
+ checkSufficientFields = false;
+ }
+ else
+ {
+ checkSufficientFields = true;
+ HashSet<string> sf = new HashSet<string>(suff.Split(new char[] { ',' }).TrimEnd());
+ for (int i = 0; i < fieldsToWrite.Length; i++)
+ {
+ if (sf.Contains(fieldsToWrite[i]))
+ {
+ sufficientFields[i] = true;
+ }
+ }
+ }
+
+ WriteHeader(lineFileOut);
+ }
+
+ /// <summary>
+ /// Write header to the lines file - indicating how to read the file later.
+ /// </summary>
+ protected virtual void WriteHeader(TextWriter @out)
+ {
+ StringBuilder sb = threadBuffer.Value;
+ if (sb == null)
+ {
+ sb = new StringBuilder();
+ threadBuffer.Value = sb;
+ }
+ sb.Length = 0;
+ sb.Append(FIELDS_HEADER_INDICATOR);
+ foreach (string f in fieldsToWrite)
+ {
+ sb.Append(SEP).Append(f);
+ }
+ @out.WriteLine(sb.ToString());
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return "Wrote " + recsCount + " line docs";
+ }
+
+ public override int DoLogic()
+ {
+ Document doc = docSize > 0 ? docMaker.MakeDocument(docSize) : docMaker.MakeDocument();
+
+ Regex matcher = threadNormalizer.Value;
+ if (matcher == null)
+ {
+ matcher = new Regex("[\t\r\n]+");
+ threadNormalizer.Value = matcher;
+ }
+
+ StringBuilder sb = threadBuffer.Value;
+ if (sb == null)
+ {
+ sb = new StringBuilder();
+ threadBuffer.Value = sb;
+ }
+ sb.Length = 0;
+
+ bool sufficient = !checkSufficientFields;
+ for (int i = 0; i < fieldsToWrite.Length; i++)
+ {
+ IIndexableField f = doc.GetField(fieldsToWrite[i]);
+ string text = f == null ? "" : matcher.Replace(f.GetStringValue(), " ").Trim();
+ sb.Append(text).Append(SEP);
+ sufficient |= text.Length > 0 && sufficientFields[i];
+ }
+ if (sufficient)
+ {
+ sb.Length = sb.Length - 1; // remove redundant last separator
+ // lineFileOut is a PrintWriter, which synchronizes internally in println.
+ LineFileOut(doc).WriteLine(sb.ToString());
+ }
+
+ return 1;
+ }
+
+ /// <summary>
+ /// Selects output line file by written doc.
+ /// Default: original output line file.
+ /// </summary>
+ protected virtual TextWriter LineFileOut(Document doc)
+ {
+ return lineFileOut;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ lineFileOut.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ /// <summary>
+ /// Set the params (docSize only)
+ /// </summary>
+ /// <param name="params">docSize, or 0 for no limit.</param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ docSize = (int)float.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
new file mode 100644
index 0000000..7e6523e
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
@@ -0,0 +1,459 @@
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Support;
+using Lucene.Net.Support.IO;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test algorithm, as read from file
+ /// </summary>
+ public class Algorithm
+ {
+ private TaskSequence sequence;
+ private readonly string[] taskPackages;
+
+ /// <summary>
+ /// Read algorithm from file.
+ /// Property examined: alt.tasks.packages == comma separated list of
+ /// alternate Assembly names where tasks would be searched for, when not found
+ /// in the default Assembly (that of <see cref="PerfTask"/>).
+ /// If the same task class appears in more than one Assembly, the Assembly
+ /// indicated first in this list will be used.
+ /// <para/>
+ /// The Lucene.Net implementation differs from Lucene in that all
+ /// referenced assemblies are also scanned for the type. However,
+ /// alt.tasks.packages may be included for assemblies that are
+ /// not referenced in your project.
+ /// </summary>
+ /// <param name="runData">perf-run-data used at running the tasks.</param>
+ /// <exception cref="Exception">if errors while parsing the algorithm.</exception>
+ public Algorithm(PerfRunData runData)
+ {
+ Config config = runData.Config;
+ taskPackages = InitTasksPackages(config);
+ string algTxt = config.AlgorithmText;
+ sequence = new TaskSequence(runData, null, null, false);
+ TaskSequence currSequence = sequence;
+ PerfTask prevTask = null;
+ StreamTokenizer stok = new StreamTokenizer(new StringReader(algTxt));
+ stok.CommentChar('#');
+ stok.IsEOLSignificant = false;
+ stok.QuoteChar('"');
+ stok.QuoteChar('\'');
+ stok.OrdinaryChar('/');
+ stok.OrdinaryChar('(');
+ stok.OrdinaryChar(')');
+ bool colonOk = false;
+ bool isDisableCountNextTask = false; // only for primitive tasks
+ currSequence.Depth = 0;
+
+ while (stok.NextToken() != StreamTokenizer.TT_EOF)
+ {
+ switch (stok.TokenType)
+ {
+
+ case StreamTokenizer.TT_WORD:
+ string s = stok.StringValue;
+ PerfTask task = (PerfTask)Activator.CreateInstance(TaskClass(config, s), runData);
+ task.AlgLineNum = stok.LineNumber;
+ task.DisableCounting = isDisableCountNextTask;
+ isDisableCountNextTask = false;
+ currSequence.AddTask(task);
+ if (task is RepSumByPrefTask)
+ {
+ stok.NextToken();
+ string prefix = stok.StringValue;
+ if (prefix == null || prefix.Length == 0)
+ {
+ throw new Exception("named report prefix problem - " + stok.ToString());
+ }
+ ((RepSumByPrefTask)task).SetPrefix(prefix);
+ }
+ // check for task param: '(' someParam ')'
+ stok.NextToken();
+ if (stok.TokenType != '(')
+ {
+ stok.PushBack();
+ }
+ else
+ {
+ // get params, for tasks that supports them - allow recursive parenthetical expressions
+ stok.IsEOLSignificant = true; // Allow params tokenizer to keep track of line number
+ StringBuilder @params = new StringBuilder();
+ stok.NextToken();
+ if (stok.TokenType != ')')
+ {
+ int count = 1;
+ while (true)
+ {
+ switch (stok.TokenType)
+ {
+ case StreamTokenizer.TT_NUMBER:
+ {
+ @params.Append(stok.NumberValue);
+ break;
+ }
+ case StreamTokenizer.TT_WORD:
+ {
+ @params.Append(stok.StringValue);
+ break;
+ }
+ case StreamTokenizer.TT_EOF:
+ {
+ throw new Exception("Unexpexted EOF: - " + stok.ToString());
+ }
+ case '"':
+ case '\'':
+ {
+ @params.Append((char)stok.TokenType);
+ // re-escape delimiters, if any
+ @params.Append(stok.StringValue.Replace("" + (char)stok.TokenType, @"\" + (char)stok.TokenType));
+ @params.Append((char)stok.TokenType);
+ break;
+ }
+ case '(':
+ {
+ @params.Append((char)stok.TokenType);
+ ++count;
+ break;
+ }
+ case ')':
+ {
+ if (--count >= 1)
+ { // exclude final closing parenthesis
+ @params.Append((char)stok.TokenType);
+ }
+ else
+ {
+ goto BALANCED_PARENS_BREAK;
+ }
+ break;
+ }
+ default:
+ {
+ @params.Append((char)stok.TokenType);
+ break;
+ }
+ }
+ stok.NextToken();
+ }
+ BALANCED_PARENS_BREAK: { }
+ }
+ stok.IsEOLSignificant = false;
+ string prm = @params.ToString().Trim();
+ if (prm.Length > 0)
+ {
+ task.SetParams(prm);
+ }
+ }
+
+ // ---------------------------------------
+ colonOk = false; prevTask = task;
+ break;
+
+ default:
+ char c = (char)stok.TokenType;
+
+ switch (c)
+ {
+
+ case ':':
+ if (!colonOk) throw new Exception("colon unexpexted: - " + stok.ToString());
+ colonOk = false;
+ // get repetitions number
+ stok.NextToken();
+ if ((char)stok.TokenType == '*')
+ {
+ ((TaskSequence)prevTask).SetRepetitions(TaskSequence.REPEAT_EXHAUST);
+ }
+ else
+ {
+ if (stok.TokenType != StreamTokenizer.TT_NUMBER)
+ {
+ throw new Exception("expected repetitions number or XXXs: - " + stok.ToString());
+ }
+ else
+ {
+ double num = stok.NumberValue;
+ stok.NextToken();
+ if (stok.TokenType == StreamTokenizer.TT_WORD && stok.StringValue.Equals("s", StringComparison.Ordinal))
+ {
+ ((TaskSequence)prevTask).SetRunTime(num);
+ }
+ else
+ {
+ stok.PushBack();
+ ((TaskSequence)prevTask).SetRepetitions((int)num);
+ }
+ }
+ }
+ // check for rate specification (ops/min)
+ stok.NextToken();
+ if (stok.TokenType != ':')
+ {
+ stok.PushBack();
+ }
+ else
+ {
+ // get rate number
+ stok.NextToken();
+ if (stok.TokenType != StreamTokenizer.TT_NUMBER) throw new Exception("expected rate number: - " + stok.ToString());
+ // check for unit - min or sec, sec is default
+ stok.NextToken();
+ if (stok.TokenType != '/')
+ {
+ stok.PushBack();
+ ((TaskSequence)prevTask).SetRate((int)stok.NumberValue, false); // set rate per sec
+ }
+ else
+ {
+ stok.NextToken();
+ if (stok.TokenType != StreamTokenizer.TT_WORD) throw new Exception("expected rate unit: 'min' or 'sec' - " + stok.ToString());
+ string unit = stok.StringValue.ToLowerInvariant();
+ if ("min".Equals(unit, StringComparison.Ordinal))
+ {
+ ((TaskSequence)prevTask).SetRate((int)stok.NumberValue, true); // set rate per min
+ }
+ else if ("sec".Equals(unit, StringComparison.Ordinal))
+ {
+ ((TaskSequence)prevTask).SetRate((int)stok.NumberValue, false); // set rate per sec
+ }
+ else
+ {
+ throw new Exception("expected rate unit: 'min' or 'sec' - " + stok.ToString());
+ }
+ }
+ }
+ colonOk = false;
+ break;
+
+ case '{':
+ case '[':
+ // a sequence
+ // check for sequence name
+ string name = null;
+ stok.NextToken();
+ if (stok.TokenType != '"')
+ {
+ stok.PushBack();
+ }
+ else
+ {
+ name = stok.StringValue;
+ if (stok.TokenType != '"' || name == null || name.Length == 0)
+ {
+ throw new Exception("sequence name problem - " + stok.ToString());
+ }
+ }
+ // start the sequence
+ TaskSequence seq2 = new TaskSequence(runData, name, currSequence, c == '[');
+ currSequence.AddTask(seq2);
+ currSequence = seq2;
+ colonOk = false;
+ break;
+
+ case '&':
+ if (currSequence.IsParallel)
+ {
+ throw new Exception("Can only create background tasks within a serial task");
+ }
+ stok.NextToken();
+ int deltaPri;
+ if (stok.TokenType != StreamTokenizer.TT_NUMBER)
+ {
+ stok.PushBack();
+ deltaPri = 0;
+ }
+ else
+ {
+ // priority
+ deltaPri = (int)stok.NumberValue;
+ }
+
+ if (prevTask == null)
+ {
+ throw new Exception("& was unexpected");
+ }
+ else if (prevTask.RunInBackground)
+ {
+ throw new Exception("double & was unexpected");
+ }
+ else
+ {
+ prevTask.SetRunInBackground(deltaPri);
+ }
+ break;
+
+ case '>':
+ currSequence.SetNoChildReport(); /* intentional fallthrough */
+ // end sequence
+ colonOk = true; prevTask = currSequence;
+ currSequence = currSequence.Parent;
+ break;
+ case '}':
+ case ']':
+ // end sequence
+ colonOk = true; prevTask = currSequence;
+ currSequence = currSequence.Parent;
+ break;
+
+ case '-':
+ isDisableCountNextTask = true;
+ break;
+
+ } //switch(c)
+ break;
+
+ } //switch(stok.ttype)
+
+ }
+
+ if (sequence != currSequence)
+ {
+ throw new Exception("Unmatched sequences");
+ }
+
+ // remove redundant top level enclosing sequences
+ while (sequence.IsCollapsable && sequence.Repetitions == 1 && sequence.GetRate() == 0)
+ {
+ IList<PerfTask> t = sequence.Tasks;
+ if (t != null && t.Count == 1)
+ {
+ PerfTask p = t[0];
+ if (p is TaskSequence)
+ {
+ sequence = (TaskSequence)p;
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private string[] InitTasksPackages(Config config)
+ {
+ // LUCENENET specific - changing the logic a bit
+ // to add all referenced assemblies by default.
+ // The alt.tasks.packages parameter still exists, but
+ // it is only necessary for assemblies that are not
+ // referenced by the host assembly.
+
+ HashSet<string> result = new HashSet<string>();
+ string alts = config.Get("alt.tasks.packages", null);
+ string dfltPkg = typeof(PerfTask).GetTypeInfo().Assembly.GetName().Name;
+ string[] referencedAssemblies = AssemblyUtils.GetReferencedAssemblies().Select(a => a.GetName().Name).ToArray();
+ result.Add(dfltPkg);
+
+ if (alts == null)
+ {
+ result.UnionWith(referencedAssemblies);
+ return result.ToArray();
+ }
+
+ foreach (string alt in alts.Split(',').TrimEnd())
+ {
+ result.Add(alt);
+ }
+ result.UnionWith(referencedAssemblies);
+ return result.ToArray();
+ }
+
+ private Type TaskClass(Config config, string taskName)
+ {
+ foreach (string pkg in taskPackages)
+ {
+ Type result = LoadType(pkg, taskName + "Task");
+ if (result != null)
+ {
+ return result;
+ }
+ }
+ // can only get here if failed to instantiate
+ throw new TypeLoadException(taskName + " not found in packages " + Collections.ToString(taskPackages));
+ }
+
+ private Type LoadType(string assemblyName, string typeName)
+ {
+ return Assembly.Load(assemblyName).DefinedTypes.FirstOrDefault(t => t.Name == typeName);
+ }
+
+ public override string ToString()
+ {
+ string newline = Environment.NewLine;
+ StringBuilder sb = new StringBuilder();
+ sb.Append(sequence.ToString());
+ sb.Append(newline);
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Execute this algorithm.
+ /// </summary>
+ public virtual void Execute()
+ {
+ try
+ {
+ sequence.RunAndMaybeStats(true);
+ }
+ finally
+ {
+ sequence.Dispose();
+ }
+ }
+
+ /// <summary>
+ /// Expert: for test purposes, return all tasks participating in this algorithm.
+ /// </summary>
+ /// <returns>All tasks participating in this algorithm.</returns>
+ public virtual IList<PerfTask> ExtractTasks()
+ {
+ List<PerfTask> res = new List<PerfTask>();
+ ExtractTasks(res, sequence);
+ return res;
+ }
+
+ private void ExtractTasks(IList<PerfTask> extrct, TaskSequence seq)
+ {
+ if (seq == null)
+ return;
+ extrct.Add(seq);
+ IList<PerfTask> t = sequence.Tasks;
+ if (t == null)
+ return;
+ foreach (PerfTask p in t)
+ {
+ if (p is TaskSequence)
+ {
+ ExtractTasks(extrct, (TaskSequence)p);
+ }
+ else
+ {
+ extrct.Add(p);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs
new file mode 100644
index 0000000..63c17cb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs
@@ -0,0 +1,156 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Util;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A factory to create an analyzer.
+ /// </summary>
+ /// <seealso cref="Tasks.AnalyzerFactoryTask"/>
+ public sealed class AnalyzerFactory
+ {
+ private IList<CharFilterFactory> charFilterFactories;
+ private TokenizerFactory tokenizerFactory;
+ private IList<TokenFilterFactory> tokenFilterFactories;
+ private string name = null;
+ private int? positionIncrementGap = null;
+ private int? offsetGap = null;
+
+ public AnalyzerFactory(IList<CharFilterFactory> charFilterFactories,
+ TokenizerFactory tokenizerFactory,
+ IList<TokenFilterFactory> tokenFilterFactories)
+ {
+ this.charFilterFactories = charFilterFactories;
+ Debug.Assert(null != tokenizerFactory);
+ this.tokenizerFactory = tokenizerFactory;
+ this.tokenFilterFactories = tokenFilterFactories;
+ }
+
+ // LUCENENET TODO: Properties ?
+ public void SetName(string name)
+ {
+ this.name = name;
+ }
+
+ public void SetPositionIncrementGap(int positionIncrementGap) // LUCENENET TODO: Nullable?
+ {
+ this.positionIncrementGap = positionIncrementGap;
+ }
+
+ public void SetOffsetGap(int offsetGap) // LUCENENET TODO: Nullable?
+ {
+ this.offsetGap = offsetGap;
+ }
+
+ public Analyzer Create()
+ {
+ return new AnalyzerAnonymousHelper(this);
+ }
+
+ private class AnalyzerAnonymousHelper : Analyzer
+ {
+ private readonly AnalyzerFactory outerInstance;
+
+ public AnalyzerAnonymousHelper(AnalyzerFactory outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected override TextReader InitReader(string fieldName, TextReader reader)
+ {
+ if (outerInstance.charFilterFactories != null && outerInstance.charFilterFactories.Count > 0)
+ {
+ TextReader wrappedReader = reader;
+ foreach (CharFilterFactory charFilterFactory in outerInstance.charFilterFactories)
+ {
+ wrappedReader = charFilterFactory.Create(wrappedReader);
+ }
+ reader = wrappedReader;
+ }
+ return reader;
+ }
+
+ protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+ {
+ Tokenizer tokenizer = outerInstance.tokenizerFactory.Create(reader);
+ TokenStream tokenStream = tokenizer;
+ foreach (TokenFilterFactory filterFactory in outerInstance.tokenFilterFactories)
+ {
+ tokenStream = filterFactory.Create(tokenStream);
+ }
+ return new TokenStreamComponents(tokenizer, tokenStream);
+ }
+
+ public override int GetPositionIncrementGap(string fieldName)
+ {
+ return outerInstance.positionIncrementGap.HasValue
+ ? outerInstance.positionIncrementGap.Value
+ : base.GetPositionIncrementGap(fieldName);
+ }
+
+ public override int GetOffsetGap(string fieldName)
+ {
+ return outerInstance.offsetGap.HasValue
+ ? outerInstance.offsetGap.Value
+ : base.GetOffsetGap(fieldName);
+ }
+ }
+
+ public override string ToString()
+ {
+ StringBuilder sb = new StringBuilder("AnalyzerFactory(");
+ if (null != name)
+ {
+ sb.Append("name:");
+ sb.Append(name);
+ sb.Append(", ");
+ }
+ if (null != positionIncrementGap)
+ {
+ sb.Append("positionIncrementGap:");
+ sb.Append(positionIncrementGap);
+ sb.Append(", ");
+ }
+ if (null != offsetGap)
+ {
+ sb.Append("offsetGap:");
+ sb.Append(offsetGap);
+ sb.Append(", ");
+ }
+ foreach (CharFilterFactory charFilterFactory in charFilterFactories)
+ {
+ sb.Append(charFilterFactory);
+ sb.Append(", ");
+ }
+ sb.Append(tokenizerFactory);
+ foreach (TokenFilterFactory tokenFilterFactory in tokenFilterFactories)
+ {
+ sb.Append(", ");
+ sb.Append(tokenFilterFactory);
+ }
+ sb.Append(')');
+ return sb.ToString();
+ }
+ }
+}
[05/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/Config.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/Config.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/Config.cs
new file mode 100644
index 0000000..e3190f9
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/Config.cs
@@ -0,0 +1,559 @@
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Perf run configuration properties.
+ /// </summary>
+ /// <remarks>
+ /// Numeric property containing ":", e.g. "10:100:5" is interpreted
+ /// as array of numeric values. It is extracted once, on first use, and
+ /// maintain a round number to return the appropriate value.
+ /// <para/>
+ /// The config property "work.dir" tells where is the root of
+ /// docs data dirs and indexes dirs. It is set to either of:
+ /// <list type="bullet">
+ /// <item><description>value supplied for it in the alg file;</description></item>
+ /// <item><description>otherwise, value of environment variable "benchmark.work.dir";</description></item>
+ /// <item><description>otherwise, "work".</description></item>
+ /// </list>
+ /// </remarks>
+ public class Config
+ {
+ // For tests, if verbose is not turned on, don't print the props.
+ private static readonly bool DEFAULT_PRINT_PROPS = SystemProperties.GetPropertyAsBoolean("tests.verbose", true);
+ private static readonly string NEW_LINE = Environment.NewLine;
+
+ private int roundNumber = 0;
+ private IDictionary<string, string> props;
+ private IDictionary<string, object> valByRound = new Dictionary<string, object>();
+ private IDictionary<string, string> colForValByRound = new Dictionary<string, string>();
+ private string algorithmText;
+
+ /// <summary>
+ /// Read both algorithm and config properties.
+ /// </summary>
+ /// <param name="algReader">From where to read algorithm and config properties.</param>
+ /// <exception cref="IOException">If there is a low-level I/O error.</exception>
+ public Config(TextReader algReader)
+ {
+ // read alg file to array of lines
+ IList<string> lines = new List<string>();
+ int lastConfigLine = 0;
+ string line;
+ while ((line = algReader.ReadLine()) != null)
+ {
+ lines.Add(line);
+ if (line.IndexOf('=') > 0)
+ {
+ lastConfigLine = lines.Count;
+ }
+ }
+ algReader.Dispose();
+ // copy props lines to string
+ MemoryStream ms = new MemoryStream();
+ TextWriter writer = new StreamWriter(ms);
+ for (int i = 0; i < lastConfigLine; i++)
+ {
+ writer.WriteLine(lines[i]);
+ }
+ // read props from string
+ this.props = new Dictionary<string, string>();
+ writer.Flush();
+ ms.Position = 0;
+ props.Load(ms);
+
+ // make sure work dir is set properly
+ string temp;
+ if (!props.TryGetValue("work.dir", out temp) || temp == null)
+ {
+ props["work.dir"] = SystemProperties.GetProperty("benchmark.work.dir", "work");
+ }
+
+ if (props.TryGetValue("print.props", out temp))
+ {
+ if (temp.Equals("true", StringComparison.OrdinalIgnoreCase))
+ {
+ PrintProps();
+ }
+ }
+ else if (DEFAULT_PRINT_PROPS)
+ {
+ PrintProps();
+ }
+
+ // copy algorithm lines
+ var sb = new StringBuilder();
+ for (int i = lastConfigLine; i < lines.Count; i++)
+ {
+ sb.Append(lines[i]);
+ sb.Append(NEW_LINE);
+ }
+ algorithmText = sb.ToString();
+ }
+
+ /// <summary>
+ /// Create config without algorithm - useful for a programmatic perf test.
+ /// </summary>
+ /// <param name="props">Configuration properties.</param>
+ public Config(IDictionary<string, string> props)
+ {
+ this.props = props;
+ string temp;
+ if (props.TryGetValue("print.props", out temp))
+ {
+ if (temp.Equals("true", StringComparison.OrdinalIgnoreCase))
+ {
+ PrintProps();
+ }
+ }
+ else if (DEFAULT_PRINT_PROPS)
+ {
+ PrintProps();
+ }
+ }
+
+ private void PrintProps()
+ {
+ SystemConsole.WriteLine("------------> config properties:");
+ List<string> propKeys = new List<string>(props.Keys);
+ propKeys.Sort();
+ foreach (string propName in propKeys)
+ {
+ SystemConsole.WriteLine(propName + " = " + props[propName]);
+ }
+ SystemConsole.WriteLine("-------------------------------");
+ }
+
+ /// <summary>
+ /// Return a string property.
+ /// </summary>
+ /// <param name="name">Name of property.</param>
+ /// <param name="dflt">Default value.</param>
+ /// <returns>A string property.</returns>
+ public virtual string Get(string name, string dflt)
+ {
+ string[] vals;
+ object temp;
+ if (valByRound.TryGetValue(name, out temp) && temp != null)
+ {
+ vals = (string[])temp;
+ return vals[roundNumber % vals.Length];
+ }
+ // done if not by round
+ string sval;
+ if (!props.TryGetValue(name, out sval))
+ {
+ sval = dflt;
+ }
+ if (sval == null)
+ {
+ return null;
+ }
+ if (sval.IndexOf(":") < 0)
+ {
+ return sval;
+ }
+ else if (sval.IndexOf(":\\") >= 0 || sval.IndexOf(":/") >= 0)
+ {
+ // this previously messed up absolute path names on Windows. Assuming
+ // there is no real value that starts with \ or /
+ return sval;
+ }
+ // first time this prop is extracted by round
+ int k = sval.IndexOf(":");
+ string colName = sval.Substring(0, k - 0);
+ sval = sval.Substring(k + 1);
+ colForValByRound[name] = colName;
+ vals = PropToStringArray(sval);
+ valByRound[name] = vals;
+ return vals[roundNumber % vals.Length];
+ }
+
+ /// <summary>
+ /// Set a property.
+ /// <para/>
+ /// Note: once a multiple values property is set, it can no longer be modified.
+ /// </summary>
+ /// <param name="name">Name of property.</param>
+ /// <param name="value">Either single or multiple property value (multiple values are separated by ":")</param>
+ public virtual void Set(string name, string value)
+ {
+ object temp;
+ if (valByRound.TryGetValue(name, out temp) && temp != null)
+ {
+ throw new Exception("Cannot modify a multi value property!");
+ }
+ props[name] = value;
+ }
+
+ /// <summary>
+ /// Return an <see cref="int"/> property.
+ /// <para/>
+ /// If the property contain ":", e.g. "10:100:5", it is interpreted
+ /// as array of ints. It is extracted once, on first call
+ /// to Get() it, and a by-round-value is returned.
+ /// </summary>
+ /// <param name="name">Name of property.</param>
+ /// <param name="dflt">Default value.</param>
+ /// <returns>An <see cref="int"/> property.</returns>
+ public virtual int Get(string name, int dflt)
+ {
+ // use value by round if already parsed
+ int[] vals;
+ object temp;
+ if (valByRound.TryGetValue(name, out temp) && temp != null)
+ {
+ vals = (int[])temp;
+ return vals[roundNumber % vals.Length];
+ }
+ // done if not by round
+ string sval;
+ if (!props.TryGetValue(name, out sval))
+ {
+ sval = "" + dflt;
+ }
+ if (sval.IndexOf(":") < 0)
+ {
+ return int.Parse(sval, CultureInfo.InvariantCulture);
+ }
+ // first time this prop is extracted by round
+ int k = sval.IndexOf(":");
+ string colName = sval.Substring(0, k - 0);
+ sval = sval.Substring(k + 1);
+ colForValByRound[name] = colName;
+ vals = PropToInt32Array(sval);
+ valByRound[name] = vals;
+ return vals[roundNumber % vals.Length];
+ }
+
+ /// <summary>
+ /// Return a double property.
+ /// <para/>
+ /// If the property contain ":", e.g. "10:100:5", it is interpreted
+ /// as array of doubles. It is extracted once, on first call
+ /// to Get() it, and a by-round-value is returned.
+ /// </summary>
+ /// <param name="name">Name of property.</param>
+ /// <param name="dflt">Default value.</param>
+ /// <returns>A double property.</returns>
+ public virtual double Get(string name, double dflt)
+ {
+ // use value by round if already parsed
+ double[] vals;
+ object temp;
+ if (valByRound.TryGetValue(name, out temp) && temp != null)
+ {
+ vals = (double[])temp;
+ return vals[roundNumber % vals.Length];
+ }
+ // done if not by round
+ string sval;
+ if (!props.TryGetValue(name, out sval))
+ {
+ sval = "" + dflt;
+ }
+ if (sval.IndexOf(":") < 0)
+ {
+ return double.Parse(sval, CultureInfo.InvariantCulture);
+ }
+ // first time this prop is extracted by round
+ int k = sval.IndexOf(":");
+ string colName = sval.Substring(0, k - 0);
+ sval = sval.Substring(k + 1);
+ colForValByRound[name] = colName;
+ vals = PropToDoubleArray(sval);
+ valByRound[name] = vals;
+ return vals[roundNumber % vals.Length];
+ }
+
+ /// <summary>
+ /// Return a boolean property.
+ /// If the property contain ":", e.g. "true.true.false", it is interpreted
+ /// as array of booleans. It is extracted once, on first call
+ /// to Get() it, and a by-round-value is returned.
+ /// </summary>
+ /// <param name="name">Name of property.</param>
+ /// <param name="dflt">Default value.</param>
+ /// <returns>A <see cref="bool"/> property.</returns>
+ public virtual bool Get(string name, bool dflt)
+ {
+ // use value by round if already parsed
+ bool[] vals;
+ object temp;
+ if (valByRound.TryGetValue(name, out temp) && temp != null)
+ {
+ vals = (bool[])temp;
+ return vals[roundNumber % vals.Length];
+ }
+ // done if not by round
+ string sval;
+ if (!props.TryGetValue(name, out sval))
+ {
+ sval = "" + dflt;
+ }
+ if (sval.IndexOf(":") < 0)
+ {
+ return bool.Parse(sval);
+ }
+ // first time this prop is extracted by round
+ int k = sval.IndexOf(":");
+ string colName = sval.Substring(0, k - 0);
+ sval = sval.Substring(k + 1);
+ colForValByRound[name] = colName;
+ vals = PropToBooleanArray(sval);
+ valByRound[name] = vals;
+ return vals[roundNumber % vals.Length];
+ }
+
+ /// <summary>
+ /// Increment the round number, for config values that are extracted by round number.
+ /// </summary>
+ /// <returns>The new round number.</returns>
+ public virtual int NewRound()
+ {
+ roundNumber++;
+
+ StringBuilder sb = new StringBuilder("--> Round ").Append(roundNumber - 1).Append("-->").Append(roundNumber);
+
+ // log changes in values
+ if (valByRound.Count > 0)
+ {
+ sb.Append(": ");
+ foreach (string name in valByRound.Keys)
+ {
+ object a = valByRound[name];
+ if (a is int[])
+ {
+ int[] ai = (int[])a;
+ int n1 = (roundNumber - 1) % ai.Length;
+ int n2 = roundNumber % ai.Length;
+ sb.Append(" ").Append(name).Append(":").Append(ai[n1]).Append("-->").Append(ai[n2]);
+ }
+ else if (a is double[])
+ {
+ double[] ad = (double[])a;
+ int n1 = (roundNumber - 1) % ad.Length;
+ int n2 = roundNumber % ad.Length;
+ sb.Append(" ").Append(name).Append(":").Append(ad[n1]).Append("-->").Append(ad[n2]);
+ }
+ else if (a is string[])
+ {
+ string[] ad = (string[])a;
+ int n1 = (roundNumber - 1) % ad.Length;
+ int n2 = roundNumber % ad.Length;
+ sb.Append(" ").Append(name).Append(":").Append(ad[n1]).Append("-->").Append(ad[n2]);
+ }
+ else
+ {
+ bool[] ab = (bool[])a;
+ int n1 = (roundNumber - 1) % ab.Length;
+ int n2 = roundNumber % ab.Length;
+ sb.Append(" ").Append(name).Append(":").Append(ab[n1]).Append("-->").Append(ab[n2]);
+ }
+ }
+ }
+
+ SystemConsole.WriteLine();
+ SystemConsole.WriteLine(sb.ToString());
+ SystemConsole.WriteLine();
+
+ return roundNumber;
+ }
+
+ private string[] PropToStringArray(string s)
+ {
+ if (s.IndexOf(":") < 0)
+ {
+ return new string[] { s };
+ }
+
+ List<string> a = new List<string>();
+ StringTokenizer st = new StringTokenizer(s, ":");
+ while (st.HasMoreTokens())
+ {
+ string t = st.NextToken();
+ a.Add(t);
+ }
+ return a.ToArray();
+ }
+
+ // extract properties to array, e.g. for "10:100:5" return int[]{10,100,5}.
+ private int[] PropToInt32Array(string s)
+ {
+ if (s.IndexOf(":") < 0)
+ {
+ return new int[] { int.Parse(s, CultureInfo.InvariantCulture) };
+ }
+
+ List<int> a = new List<int>();
+ StringTokenizer st = new StringTokenizer(s, ":");
+ while (st.HasMoreTokens())
+ {
+ string t = st.NextToken();
+ a.Add(int.Parse(t, CultureInfo.InvariantCulture));
+ }
+ int[] res = new int[a.Count];
+ for (int i = 0; i < a.Count; i++)
+ {
+ res[i] = a[i];
+ }
+ return res;
+ }
+
+ // extract properties to array, e.g. for "10.7:100.4:-2.3" return int[]{10.7,100.4,-2.3}.
+ private double[] PropToDoubleArray(string s)
+ {
+ if (s.IndexOf(":") < 0)
+ {
+ return new double[] { double.Parse(s, CultureInfo.InvariantCulture) };
+ }
+
+ List<double> a = new List<double>();
+ StringTokenizer st = new StringTokenizer(s, ":");
+ while (st.HasMoreTokens())
+ {
+ string t = st.NextToken();
+ a.Add(double.Parse(t, CultureInfo.InvariantCulture));
+ }
+ double[] res = new double[a.Count];
+ for (int i = 0; i < a.Count; i++)
+ {
+ res[i] = a[i];
+ }
+ return res;
+ }
+
+ // extract properties to array, e.g. for "true:true:false" return boolean[]{true,false,false}.
+ private bool[] PropToBooleanArray(string s)
+ {
+ if (s.IndexOf(":") < 0)
+ {
+ return new bool[] { bool.Parse(s) };
+ }
+
+ List<bool> a = new List<bool>();
+ StringTokenizer st = new StringTokenizer(s, ":");
+ while (st.HasMoreTokens())
+ {
+ string t = st.NextToken();
+ a.Add(bool.Parse(t));
+ }
+ bool[] res = new bool[a.Count];
+ for (int i = 0; i < a.Count; i++)
+ {
+ res[i] = a[i];
+ }
+ return res;
+ }
+
+ /// <summary>
+ /// Gets names of params set by round, for reports title.
+ /// </summary>
+ public virtual string GetColsNamesForValsByRound()
+ {
+ if (colForValByRound.Count == 0)
+ {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ foreach (string name in colForValByRound.Keys)
+ {
+ string colName = colForValByRound[name];
+ sb.Append(" ").Append(colName);
+ }
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Gets values of params set by round, for reports lines.
+ /// </summary>
+ public virtual string GetColsValuesForValsByRound(int roundNum)
+ {
+ if (colForValByRound.Count == 0)
+ {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ foreach (string name in colForValByRound.Keys)
+ {
+ string colName = colForValByRound[name];
+ string template = " " + colName;
+ if (roundNum < 0)
+ {
+ // just append blanks
+ sb.Append(Formatter.FormatPaddLeft("-", template));
+ }
+ else
+ {
+ // append actual values, for that round
+ object a;
+ valByRound.TryGetValue(name, out a);
+ if (a is int[])
+ {
+ int[] ai = (int[])a;
+ int n = roundNum % ai.Length;
+ sb.Append(Formatter.Format(ai[n], template));
+ }
+ else if (a is double[])
+ {
+ double[] ad = (double[])a;
+ int n = roundNum % ad.Length;
+ sb.Append(Formatter.Format(2, ad[n], template));
+ }
+ else if (a is string[])
+ {
+ string[] ad = (string[])a;
+ int n = roundNum % ad.Length;
+ sb.Append(ad[n]);
+ }
+ else
+ {
+ bool[] ab = (bool[])a;
+ int n = roundNum % ab.Length;
+ sb.Append(Formatter.FormatPaddLeft("" + ab[n], template));
+ }
+ }
+ }
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Gets the round number.
+ /// </summary>
+ public virtual int RoundNumber
+ {
+ get { return roundNumber; }
+ }
+
+ /// <summary>
+ /// Gets the algorithmText.
+ /// </summary>
+ public virtual string AlgorithmText
+ {
+ get { return algorithmText; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/FileUtils.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/FileUtils.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/FileUtils.cs
new file mode 100644
index 0000000..3d05db8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/FileUtils.cs
@@ -0,0 +1,46 @@
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// File utilities.
+ /// </summary>
+ public class FileUtils
+ {
+ /// <summary>
+ /// Delete files and directories, even if non-empty.
+ /// </summary>
+ /// <param name="dir">File or directory.</param>
+ /// <returns><c>true</c> on success, <c>false</c> if no or part of files have been deleted.</returns>
+ /// <exception cref="IOException">If there is a low-level I/O error.</exception>
+ public static bool FullyDelete(DirectoryInfo dir)
+ {
+ try
+ {
+ Directory.Delete(dir.FullName, true);
+ return true;
+ }
+ catch
+ {
+ return !Directory.Exists(dir.FullName);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/Format.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/Format.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/Format.cs
new file mode 100644
index 0000000..85f1fdd
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/Format.cs
@@ -0,0 +1,109 @@
+using System;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Formatting utilities (for reports).
+ /// </summary>
+ public class Formatter // LUCENENET specific - renamed from Format because of method name collision
+ {
+ private static string[] numFormat = {
+ "N0",
+ "N1",
+ "N2"
+ };
+
+ private static readonly string padd = " ";
+
+ /// <summary>
+ /// Padd a number from left.
+ /// </summary>
+ /// <param name="numFracDigits">Number of digits in fraction part - must be 0 or 1 or 2.</param>
+ /// <param name="f">Number to be formatted.</param>
+ /// <param name="col">Column name (used for deciding on length).</param>
+ /// <returns>Formatted string.</returns>
+ public static string Format(int numFracDigits, float f, string col)
+ {
+ string res = padd + string.Format(numFormat[numFracDigits], f);
+ return res.Substring(res.Length - col.Length);
+ }
+
+ public static string Format(int numFracDigits, double f, string col)
+ {
+ string res = padd + string.Format(numFormat[numFracDigits], f);
+ return res.Substring(res.Length - col.Length);
+ }
+
+ /// <summary>
+ /// Pad a number from right.
+ /// </summary>
+ /// <param name="numFracDigits">Number of digits in fraction part - must be 0 or 1 or 2.</param>
+ /// <param name="f">Number to be formatted.</param>
+ /// <param name="col">Column name (used for deciding on length).</param>
+ /// <returns>Formatted string.</returns>
+ public static string FormatPaddRight(int numFracDigits, float f, string col)
+ {
+ string res = string.Format(numFormat[numFracDigits], f) + padd;
+ return res.Substring(0, col.Length - 0);
+ }
+
+ public static string FormatPaddRight(int numFracDigits, double f, string col)
+ {
+ string res = string.Format(numFormat[numFracDigits], f) + padd;
+ return res.Substring(0, col.Length - 0);
+ }
+
+ /// <summary>
+ /// Pad a number from left.
+ /// </summary>
+ /// <param name="n">Number to be formatted.</param>
+ /// <param name="col">Column name (used for deciding on length).</param>
+ /// <returns>Formatted string.</returns>
+ public static string Format(int n, string col)
+ {
+ string res = padd + n;
+ return res.Substring(res.Length - col.Length);
+ }
+
+ /// <summary>
+ /// Pad a string from right.
+ /// </summary>
+ /// <param name="s">String to be formatted.</param>
+ /// <param name="col">Column name (used for deciding on length).</param>
+ /// <returns>Formatted string.</returns>
+ public static string Format(string s, string col)
+ {
+ string s1 = (s + padd);
+ return s1.Substring(0, Math.Min(col.Length, s1.Length));
+ }
+
+ /// <summary>
+ /// Pad a string from left.
+ /// </summary>
+ /// <param name="s">String to be formatted.</param>
+ /// <param name="col">Column name (used for deciding on length).</param>
+ /// <returns>Formatted string.</returns>
+ public static string FormatPaddLeft(string s, string col)
+ {
+ string res = padd + s;
+ return res.Substring(res.Length - col.Length);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/StreamUtils.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/StreamUtils.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/StreamUtils.cs
new file mode 100644
index 0000000..cf57512
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/StreamUtils.cs
@@ -0,0 +1,132 @@
+using ICSharpCode.SharpZipLib.BZip2;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Stream utilities.
+ /// </summary>
+ public class StreamUtils
+ {
+ /// <summary>Buffer size used across the benchmark package</summary>
+ public static readonly int BUFFER_SIZE = 1 << 16; // 64K
+
+ // LUCENENET specific - de-nested Type and renamed FileType
+
+ private static readonly IDictionary<string, FileType?> extensionToType = new Dictionary<string, FileType?>();
+ static StreamUtils()
+ {
+ // these in are lower case, we will lower case at the test as well
+ extensionToType[".bz2"] = FileType.BZIP2;
+ extensionToType[".bzip"] = FileType.BZIP2;
+ extensionToType[".gz"] = FileType.GZIP;
+ extensionToType[".gzip"] = FileType.GZIP;
+ }
+
+ /// <summary>
+ /// Returns an <see cref="Stream"/> over the requested file. This method
+ /// attempts to identify the appropriate <see cref="Stream"/> instance to return
+ /// based on the file name (e.g., if it ends with .bz2 or .bzip, return a
+ /// 'bzip' <see cref="Stream"/>).
+ /// </summary>
+ public static Stream GetInputStream(FileInfo file)
+ {
+ // First, create a FileInputStream, as this will be required by all types.
+ // Wrap with BufferedInputStream for better performance
+ Stream @in = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
+ return GetFileType(file).GetInputStream(@in);
+ }
+
+ /// <summary>Return the type of the file, or <c>null</c> if unknown.</summary>
+ private static FileType GetFileType(FileInfo file)
+ {
+ FileType? type = null;
+ string fileName = file.Name;
+ int idx = fileName.LastIndexOf('.');
+ if (idx != -1)
+ {
+ extensionToType.TryGetValue(fileName.Substring(idx).ToLowerInvariant(), out type);
+ }
+ return !type.HasValue ? FileType.PLAIN : type.Value;
+ }
+
+ /// <summary>
+ /// Returns an <see cref="Stream"/> over the requested file, identifying
+ /// the appropriate <see cref="Stream"/> instance similar to <see cref="GetInputStream(FileInfo)"/>.
+ /// </summary>
+ public static Stream GetOutputStream(FileInfo file)
+ {
+ // First, create a FileInputStream, as this will be required by all types.
+ // Wrap with BufferedInputStream for better performance
+ Stream os = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
+ return GetFileType(file).GetOutputStream(os);
+ }
+ }
+
+ /// <summary>File format type.</summary>
+ public enum FileType
+ {
+ /// <summary>
+ /// BZIP2 is automatically used for <b>.bz2</b> and <b>.bzip2</b> extensions.
+ /// </summary>
+ BZIP2,
+
+ /// <summary>
+ /// GZIP is automatically used for <b>.gz</b> and <b>.gzip</b> extensions.
+ /// </summary>
+ GZIP,
+
+ /// <summary>
+ /// Plain text is used for anything which is not GZIP or BZIP.
+ /// </summary>
+ PLAIN
+ }
+
+ internal static class FileTypeExtensions
+ {
+ public static Stream GetInputStream(this FileType fileType, Stream input)
+ {
+ switch (fileType)
+ {
+ case FileType.BZIP2:
+ return new BZip2InputStream(input);
+ case FileType.GZIP:
+ return new GZipStream(input, CompressionMode.Decompress);
+ default:
+ return input;
+ }
+ }
+
+ public static Stream GetOutputStream(this FileType fileType, Stream output)
+ {
+ switch (fileType)
+ {
+ case FileType.BZIP2:
+ return new BZip2OutputStream(output);
+ case FileType.GZIP:
+ return new GZipStream(output, CompressionMode.Compress);
+ default:
+ return output;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Constants.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Constants.cs b/src/Lucene.Net.Benchmark/Constants.cs
new file mode 100644
index 0000000..72bdbdc
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Constants.cs
@@ -0,0 +1,33 @@
+namespace Lucene.Net.Benchmarks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Various benchmarking constants (mostly defaults)
+ /// </summary>
+ public class Constants
+ {
+ public static readonly int DEFAULT_RUN_COUNT = 5;
+ public static readonly int DEFAULT_SCALE_UP = 5;
+ public static readonly int DEFAULT_LOG_STEP = 1000;
+
+ public static bool[] BOOLEANS = new bool[] { false, true };
+
+ public static readonly int DEFAULT_MAXIMUM_DOCUMENTS = int.MaxValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
new file mode 100644
index 0000000..0241099
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj
@@ -0,0 +1,214 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+ <ProjectGuid>{EDC77CB4-597F-4818-8C83-3C006D12C384}</ProjectGuid>
+ <OutputType>Library</OutputType>
+ <AppDesignerFolder>Properties</AppDesignerFolder>
+ <RootNamespace>Lucene.Net.Benchmarks</RootNamespace>
+ <AssemblyName>Lucene.Net.Benchmark</AssemblyName>
+ <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
+ <FileAlignment>512</FileAlignment>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+ <DebugSymbols>true</DebugSymbols>
+ <DebugType>full</DebugType>
+ <Optimize>false</Optimize>
+ <OutputPath>bin\Debug\</OutputPath>
+ <DefineConstants>DEBUG;TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ <DocumentationFile>
+ </DocumentationFile>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
+ <OutputPath>bin\Release\</OutputPath>
+ <DefineConstants>TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <PropertyGroup>
+ <DefineConstants>$(DefineConstants);FEATURE_SERIALIZABLE</DefineConstants>
+ </PropertyGroup>
+ <ItemGroup>
+ <Reference Include="System" />
+ <Reference Include="System.Core" />
+ <Reference Include="Microsoft.CSharp" />
+ </ItemGroup>
+ <ItemGroup>
+ <Compile Include="ByTask\Benchmark.cs" />
+ <Compile Include="ByTask\Feeds\AbstractQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\ContentItemsSource.cs" />
+ <Compile Include="ByTask\Feeds\ContentSource.cs" />
+ <Compile Include="ByTask\Feeds\DemoHTMLParser.cs" />
+ <Compile Include="ByTask\Feeds\DirContentSource.cs" />
+ <Compile Include="ByTask\Feeds\DocData.cs" />
+ <Compile Include="ByTask\Feeds\DocMaker.cs" />
+ <Compile Include="ByTask\Feeds\EnwikiContentSource.cs" />
+ <Compile Include="ByTask\Feeds\EnwikiQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\FacetSource.cs" />
+ <Compile Include="ByTask\Feeds\FileBasedQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\GeonamesLineParser.cs" />
+ <Compile Include="ByTask\Feeds\HTMLParser.cs" />
+ <Compile Include="ByTask\Feeds\LineDocSource.cs" />
+ <Compile Include="ByTask\Feeds\LongToEnglishContentSource.cs" />
+ <Compile Include="ByTask\Feeds\LongToEnglishQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\NoMoreDataException.cs" />
+ <Compile Include="ByTask\Feeds\QueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\RandomFacetSource.cs" />
+ <Compile Include="ByTask\Feeds\ReutersContentSource.cs" />
+ <Compile Include="ByTask\Feeds\ReutersQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\SimpleQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\SimpleSloppyPhraseQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\SingleDocSource.cs" />
+ <Compile Include="ByTask\Feeds\SortableSingleDocSource.cs" />
+ <Compile Include="ByTask\Feeds\SpatialDocMaker.cs" />
+ <Compile Include="ByTask\Feeds\SpatialFileQueryMaker.cs" />
+ <Compile Include="ByTask\Feeds\TrecContentSource.cs" />
+ <Compile Include="ByTask\Feeds\TrecDocParser.cs" />
+ <Compile Include="ByTask\Feeds\TrecFBISParser.cs" />
+ <Compile Include="ByTask\Feeds\TrecFR94Parser.cs" />
+ <Compile Include="ByTask\Feeds\TrecFTParser.cs" />
+ <Compile Include="ByTask\Feeds\TrecGov2Parser.cs" />
+ <Compile Include="ByTask\Feeds\TrecLATimesParser.cs" />
+ <Compile Include="ByTask\Feeds\TrecParserByPath.cs" />
+ <Compile Include="ByTask\PerfRunData.cs" />
+ <Compile Include="ByTask\Programmatic\Sample.cs" />
+ <Compile Include="ByTask\Stats\Points.cs" />
+ <Compile Include="ByTask\Stats\Report.cs" />
+ <Compile Include="ByTask\Stats\TaskStats.cs" />
+ <Compile Include="ByTask\Tasks\AddDocTask.cs" />
+ <Compile Include="ByTask\Tasks\AddFacetedDocTask.cs" />
+ <Compile Include="ByTask\Tasks\AddIndexesTask.cs" />
+ <Compile Include="ByTask\Tasks\AnalyzerFactoryTask.cs" />
+ <Compile Include="ByTask\Tasks\BenchmarkHighlighter.cs" />
+ <Compile Include="ByTask\Tasks\ClearStatsTask.cs" />
+ <Compile Include="ByTask\Tasks\CloseIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\CloseReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\CloseTaxonomyIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\CloseTaxonomyReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\CommitIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\CommitTaxonomyIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\ConsumeContentSourceTask.cs" />
+ <Compile Include="ByTask\Tasks\CreateIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\CreateTaxonomyIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\ForceMergeTask.cs" />
+ <Compile Include="ByTask\Tasks\NearRealtimeReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\NewAnalyzerTask.cs" />
+ <Compile Include="ByTask\Tasks\NewCollationAnalyzerTask.cs" />
+ <Compile Include="ByTask\Tasks\NewLocaleTask.cs" />
+ <Compile Include="ByTask\Tasks\NewRoundTask.cs" />
+ <Compile Include="ByTask\Tasks\OpenIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\OpenReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\OpenTaxonomyIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\OpenTaxonomyReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\PerfTask.cs" />
+ <Compile Include="ByTask\Tasks\PrintReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\ReadTask.cs" />
+ <Compile Include="ByTask\Tasks\ReadTokensTask.cs" />
+ <Compile Include="ByTask\Tasks\ReopenReaderTask.cs" />
+ <Compile Include="ByTask\Tasks\RepAllTask.cs" />
+ <Compile Include="ByTask\Tasks\ReportTask.cs" />
+ <Compile Include="ByTask\Tasks\RepSelectByPrefTask.cs" />
+ <Compile Include="ByTask\Tasks\RepSumByNameRoundTask.cs" />
+ <Compile Include="ByTask\Tasks\RepSumByNameTask.cs" />
+ <Compile Include="ByTask\Tasks\RepSumByPrefRoundTask.cs" />
+ <Compile Include="ByTask\Tasks\RepSumByPrefTask.cs" />
+ <Compile Include="ByTask\Tasks\ResetInputsTask.cs" />
+ <Compile Include="ByTask\Tasks\ResetSystemEraseTask.cs" />
+ <Compile Include="ByTask\Tasks\ResetSystemSoftTask.cs" />
+ <Compile Include="ByTask\Tasks\RollbackIndexTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchTravRetHighlightTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchTravRetLoadFieldSelectorTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchTravRetTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchTravRetVectorHighlightTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchTravTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchWithCollectorTask.cs" />
+ <Compile Include="ByTask\Tasks\SearchWithSortTask.cs" />
+ <Compile Include="ByTask\Tasks\SetPropTask.cs" />
+ <Compile Include="ByTask\Tasks\TaskSequence.cs" />
+ <Compile Include="ByTask\Tasks\UpdateDocTask.cs" />
+ <Compile Include="ByTask\Tasks\WaitForMergesTask.cs" />
+ <Compile Include="ByTask\Tasks\WaitTask.cs" />
+ <Compile Include="ByTask\Tasks\WarmTask.cs" />
+ <Compile Include="ByTask\Tasks\WriteEnwikiLineDocTask.cs" />
+ <Compile Include="ByTask\Tasks\WriteLineDocTask.cs" />
+ <Compile Include="ByTask\Utils\Algorithm.cs" />
+ <Compile Include="ByTask\Utils\AnalyzerFactory.cs" />
+ <Compile Include="ByTask\Utils\Config.cs" />
+ <Compile Include="ByTask\Utils\FileUtils.cs" />
+ <Compile Include="ByTask\Utils\Format.cs" />
+ <Compile Include="ByTask\Utils\StreamUtils.cs" />
+ <Compile Include="Constants.cs" />
+ <Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="Quality\Judge.cs" />
+ <Compile Include="Quality\QualityBenchmark.cs" />
+ <Compile Include="Quality\QualityQuery.cs" />
+ <Compile Include="Quality\QualityQueryParser.cs" />
+ <Compile Include="Quality\QualityStats.cs" />
+ <Compile Include="Quality\Trec\QueryDriver.cs" />
+ <Compile Include="Quality\Trec\Trec1MQReader.cs" />
+ <Compile Include="Quality\Trec\TrecJudge.cs" />
+ <Compile Include="Quality\Trec\TrecTopicsReader.cs" />
+ <Compile Include="Quality\Utils\DocNameExtractor.cs" />
+ <Compile Include="Quality\Utils\QualityQueriesFinder.cs" />
+ <Compile Include="Quality\Utils\SimpleQQParser.cs" />
+ <Compile Include="Quality\Utils\SubmissionReport.cs" />
+ <Compile Include="Utils\ExtractReuters.cs" />
+ <Compile Include="Utils\ExtractWikipedia.cs" />
+ <Compile Include="..\CommonAssemblyInfo.cs">
+ <Link>Properties\CommonAssemblyInfo.cs</Link>
+ </Compile>
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\Lucene.Net.Analysis.Common\Lucene.Net.Analysis.Common.csproj">
+ <Project>{4ADD0BBC-B900-4715-9526-D871DE8EEA64}</Project>
+ <Name>Lucene.Net.Analysis.Common</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Facet\Lucene.Net.Facet.csproj">
+ <Project>{48F7884A-9454-4E88-8413-9D35992CB440}</Project>
+ <Name>Lucene.Net.Facet</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Highlighter\Lucene.Net.Highlighter.csproj">
+ <Project>{E9E769EA-8504-44BC-8DC9-CCF958765F8F}</Project>
+ <Name>Lucene.Net.Highlighter</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.ICU\Lucene.Net.ICU.csproj">
+ <Project>{349cb7c9-7534-4e1d-9b0a-5521441af0ae}</Project>
+ <Name>Lucene.Net.ICU</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Queries\Lucene.Net.Queries.csproj">
+ <Project>{69D7956C-C2CC-4708-B399-A188FEC384C4}</Project>
+ <Name>Lucene.Net.Queries</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.QueryParser\Lucene.Net.QueryParser.csproj">
+ <Project>{949BA34B-6AE6-4CE3-B578-61E13E4D76BF}</Project>
+ <Name>Lucene.Net.QueryParser</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net.Spatial\Lucene.Net.Spatial.csproj">
+ <Project>{35C347F4-24B2-4BE5-8117-A0E3001551CE}</Project>
+ <Name>Lucene.Net.Spatial</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Lucene.Net\Lucene.Net.csproj">
+ <Project>{5D4AD9BE-1FFB-41AB-9943-25737971BF57}</Project>
+ <Name>Lucene.Net</Name>
+ </ProjectReference>
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="Lucene.Net.Benchmark.project.json" />
+ </ItemGroup>
+ <ItemGroup />
+ <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+ <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Microsoft.Common.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
new file mode 100644
index 0000000..0a83392
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.project.json
@@ -0,0 +1,15 @@
+{
+ "runtimes": {
+ "win": {}
+ },
+ "dependencies": {
+ "icu.net": "54.1.1-alpha",
+ "Sax.Net": "2.0.2",
+ "SharpZipLib": "0.86.0",
+ "Spatial4n.Core": "0.4.1-beta00003",
+ "TagSoup.Net": "1.2.1.1"
+ },
+ "frameworks": {
+ "net451": {}
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Properties/AssemblyInfo.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Properties/AssemblyInfo.cs b/src/Lucene.Net.Benchmark/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..8060798
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Properties/AssemblyInfo.cs
@@ -0,0 +1,30 @@
+using System;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Lucene.Net.Benchmark")]
+[assembly: AssemblyDescription(
+ "System for benchmarking " +
+ "for the Lucene.Net full-text search engine library from The Apache Software Foundation.")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyDefaultAlias("Lucene.Net.Benchmark")]
+[assembly: AssemblyCulture("")]
+
+[assembly: CLSCompliant(true)]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("edc77cb4-597f-4818-8c83-3c006d12c384")]
+
+// for testing
+[assembly: InternalsVisibleTo("Lucene.Net.Tests.Benchmark")]
+
+// NOTE: Version information is in CommonAssemblyInfo.cs
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Judge.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Judge.cs b/src/Lucene.Net.Benchmark/Quality/Judge.cs
new file mode 100644
index 0000000..7cd2089
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Judge.cs
@@ -0,0 +1,55 @@
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.Quality
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Judge if a document is relevant for a quality query.
+ /// </summary>
+ public interface IJudge
+ {
+ /// <summary>
+ /// Judge if document <paramref name="docName"/> is relevant for the given quality query.
+ /// </summary>
+ /// <param name="docName">Name of doc tested for relevancy.</param>
+ /// <param name="query">Tested quality query.</param>
+ /// <returns><c>true</c> if relevant, <c>false</c> if not.</returns>
+ bool IsRelevant(string docName, QualityQuery query);
+
+ /// <summary>
+ /// Validate that queries and this <see cref="IJudge"/> match each other.
+ /// To be perfectly valid, this Judge must have some data for each and every
+ /// input quality query, and must not have any data on any other quality query.
+ /// <b>Note</b>: the quality benchmark run would not fail in case of imperfect
+ /// validity, just a warning message would be logged.
+ /// </summary>
+ /// <param name="qq">Quality queries to be validated.</param>
+ /// <param name="logger">If not <c>null</c>, validation issues are logged.</param>
+ /// <returns><c>true</c> if perfectly valid, <c>false</c> if not.</returns>
+ bool ValidateData(QualityQuery[] qq, TextWriter logger);
+
+ /// <summary>
+ /// Return the maximal recall for the input quality query.
+ /// It is the number of relevant docs this <see cref="IJudge"/> "knows" for the query.
+ /// </summary>
+ /// <param name="query">The query whose maximal recall is needed.</param>
+ /// <returns></returns>
+ int MaxRecall(QualityQuery query);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/QualityBenchmark.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/QualityBenchmark.cs b/src/Lucene.Net.Benchmark/Quality/QualityBenchmark.cs
new file mode 100644
index 0000000..ef53e25
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/QualityBenchmark.cs
@@ -0,0 +1,159 @@
+using Lucene.Net.Benchmarks.Quality.Utils;
+using Lucene.Net.Search;
+using System;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.Quality
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Main entry point for running a quality benchmark.
+ /// <para/>
+ /// There are two main configurations for running a quality benchmark:
+ /// <list type="bullet">
+ /// <item><description>Against existing judgements.</description></item>
+ /// <item><description>For submission (e.g. for a contest).</description></item>
+ /// </list>
+ /// The first configuration requires a non null <see cref="IJudge"/>.
+ /// The second configuration requires a non null <see cref="Utils.SubmissionReport"/>.
+ /// </summary>
+ public class QualityBenchmark
+ {
+ /// <summary>Quality Queries that this quality benchmark would execute.</summary>
+ protected QualityQuery[] m_qualityQueries;
+
+ /// <summary>Parser for turning QualityQueries into Lucene Queries.</summary>
+ protected IQualityQueryParser m_qqParser;
+
+ /// <summary>Index to be searched.</summary>
+ protected IndexSearcher m_searcher;
+
+ /// <summary>index field to extract doc name for each search result; used for judging the results.</summary>
+ protected string m_docNameField;
+
+ /// <summary>maximal number of queries that this quality benchmark runs. Default: maxint. Useful for debugging.</summary>
+ private int maxQueries = int.MaxValue;
+
+ /// <summary>Maximal number of results to collect for each query. Default: 1000.</summary>
+ private int maxResults = 1000;
+
+ /// <summary>
+ /// Create a <see cref="QualityBenchmark"/>.
+ /// </summary>
+ /// <param name="qqs">Quality queries to run.</param>
+ /// <param name="qqParser">Parser for turning QualityQueries into Lucene Queries.</param>
+ /// <param name="searcher">Index to be searched.</param>
+ /// <param name="docNameField">
+ /// Name of field containing the document name.
+ /// This allows to extract the doc name for search results,
+ /// and is important for judging the results.
+ /// </param>
+ public QualityBenchmark(QualityQuery[] qqs, IQualityQueryParser qqParser,
+ IndexSearcher searcher, string docNameField)
+ {
+ this.m_qualityQueries = qqs;
+ this.m_qqParser = qqParser;
+ this.m_searcher = searcher;
+ this.m_docNameField = docNameField;
+ }
+
+ /// <summary>
+ /// Run the quality benchmark.
+ /// </summary>
+ /// <param name="judge">
+ /// The judge that can tell if a certain result doc is relevant for a certain quality query.
+ /// If null, no judgements would be made. Usually null for a submission run.
+ /// </param>
+ /// <param name="submitRep">Submission report is created if non null.</param>
+ /// <param name="qualityLog">If not null, quality run data would be printed for each query.</param>
+ /// <returns><see cref="QualityStats"/> of each quality query that was executed.</returns>
+ /// <exception cref="Exception">If quality benchmark failed to run.</exception>
+ public virtual QualityStats[] Execute(IJudge judge, SubmissionReport submitRep,
+ TextWriter qualityLog)
+ {
+ int nQueries = Math.Min(maxQueries, m_qualityQueries.Length);
+ QualityStats[] stats = new QualityStats[nQueries];
+ for (int i = 0; i < nQueries; i++)
+ {
+ QualityQuery qq = m_qualityQueries[i];
+ // generate query
+ Query q = m_qqParser.Parse(qq);
+ // search with this query
+ long t1 = Support.Time.CurrentTimeMilliseconds();
+ TopDocs td = m_searcher.Search(q, null, maxResults);
+ long searchTime = Support.Time.CurrentTimeMilliseconds() - t1;
+ //most likely we either submit or judge, but check both
+ if (judge != null)
+ {
+ stats[i] = AnalyzeQueryResults(qq, q, td, judge, qualityLog, searchTime);
+ }
+ if (submitRep != null)
+ {
+ submitRep.Report(qq, td, m_docNameField, m_searcher);
+ }
+ }
+ if (submitRep != null)
+ {
+ submitRep.Flush();
+ }
+ return stats;
+ }
+
+ /// <summary>Analyze/judge results for a single quality query; optionally log them.</summary>
+ private QualityStats AnalyzeQueryResults(QualityQuery qq, Query q, TopDocs td, IJudge judge, TextWriter logger, long searchTime)
+ {
+ QualityStats stts = new QualityStats(judge.MaxRecall(qq), searchTime);
+ ScoreDoc[] sd = td.ScoreDocs;
+ long t1 = Support.Time.CurrentTimeMilliseconds(); // extraction of first doc name we measure also construction of doc name extractor, just in case.
+ DocNameExtractor xt = new DocNameExtractor(m_docNameField);
+ for (int i = 0; i < sd.Length; i++)
+ {
+ string docName = xt.DocName(m_searcher, sd[i].Doc);
+ long docNameExtractTime = Support.Time.CurrentTimeMilliseconds() - t1;
+ t1 = Support.Time.CurrentTimeMilliseconds();
+ bool isRelevant = judge.IsRelevant(docName, qq);
+ stts.AddResult(i + 1, isRelevant, docNameExtractTime);
+ }
+ if (logger != null)
+ {
+ logger.WriteLine(qq.QueryID + " - " + q);
+ stts.Log(qq.QueryID + " Stats:", 1, logger, " ");
+ }
+ return stts;
+ }
+
+ /// <summary>
+ /// The maximum number of quality queries to run. Useful at debugging.
+ /// </summary>
+ public virtual int MaxQueries
+ {
+ get { return maxQueries; }
+ set { maxQueries = value; }
+ }
+
+ /// <summary>
+ /// The maximum number of results to collect for each quality query.
+ /// </summary>
+ public virtual int MaxResults
+ {
+ get { return maxResults; }
+ set { maxResults = value; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/QualityQuery.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/QualityQuery.cs b/src/Lucene.Net.Benchmark/Quality/QualityQuery.cs
new file mode 100644
index 0000000..de4a945
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/QualityQuery.cs
@@ -0,0 +1,107 @@
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+
+namespace Lucene.Net.Benchmarks.Quality
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A QualityQuery has an ID and some name-value pairs.
+ /// <para/>
+ /// The ID allows to map the quality query with its judgements.
+ /// <para/>
+ /// The name-value pairs are used by a
+ /// <see cref="QualityQueryParser"/>
+ /// to create a Lucene <see cref="Search.Query"/>.
+ /// <para/>
+ /// It is very likely that name-value-pairs would be mapped into fields in a Lucene query,
+ /// but it is up to the QualityQueryParser how to map - e.g. all values in a single field,
+ /// or each pair as its own field, etc., - and this of course must match the way the
+ /// searched index was constructed.
+ /// </summary>
+ public class QualityQuery : IComparable<QualityQuery>
+ {
+ private string queryID;
+ private IDictionary<string, string> nameValPairs;
+
+ /// <summary>
+ /// Create a <see cref="QualityQuery"/> with given ID and name-value pairs.
+ /// </summary>
+ /// <param name="queryID">ID of this quality query.</param>
+ /// <param name="nameValPairs">The contents of this quality query.</param>
+ public QualityQuery(string queryID, IDictionary<string, string> nameValPairs)
+ {
+ this.queryID = queryID;
+ this.nameValPairs = nameValPairs;
+ }
+
+ /// <summary>
+ /// Return all the names of name-value-pairs in this <see cref="QualityQuery"/>.
+ /// </summary>
+ public virtual string[] GetNames()
+ {
+ return nameValPairs.Keys.ToArray();
+ }
+
+ /// <summary>
+ /// Return the value of a certain name-value pair.
+ /// </summary>
+ /// <param name="name">The name whose value should be returned.</param>
+ /// <returns></returns>
+ public virtual string GetValue(string name)
+ {
+ string result;
+ nameValPairs.TryGetValue(name, out result);
+ return result;
+ }
+
+ /// <summary>
+ /// Gets the ID of this query.
+ /// The ID allows to map the quality query with its judgements.
+ /// </summary>
+ public virtual string QueryID
+ {
+ get { return queryID; }
+ }
+
+ /// <summary>
+ /// For a nicer sort of input queries before running them.
+ /// Try first as ints, fall back to string if not int.
+ /// </summary>
+ /// <param name="other"></param>
+ /// <returns></returns>
+ public virtual int CompareTo(QualityQuery other)
+ {
+ try
+ {
+ // compare as ints when ids ints
+ int n = int.Parse(queryID, CultureInfo.InvariantCulture);
+ int nOther = int.Parse(other.queryID, CultureInfo.InvariantCulture);
+ return n - nOther;
+ }
+ catch (FormatException /*e*/)
+ {
+ // fall back to string comparison
+ return queryID.CompareToOrdinal(other.queryID);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/QualityQueryParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/QualityQueryParser.cs b/src/Lucene.Net.Benchmark/Quality/QualityQueryParser.cs
new file mode 100644
index 0000000..a62d472
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/QualityQueryParser.cs
@@ -0,0 +1,35 @@
+using Lucene.Net.Search;
+using System;
+
+namespace Lucene.Net.Benchmarks.Quality
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parse a <see cref="QualityQuery"/> into a Lucene query.
+ /// </summary>
+ public interface IQualityQueryParser
+ {
+ /// <summary>
+ /// Parse a given <see cref="QualityQuery"/> into a Lucene query.
+ /// </summary>
+ /// <param name="qq">The quality query to be parsed.</param>
+ /// <exception cref="FormatException">If parsing failed.</exception>
+ Query Parse(QualityQuery qq);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/QualityStats.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/QualityStats.cs b/src/Lucene.Net.Benchmark/Quality/QualityStats.cs
new file mode 100644
index 0000000..2098085
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/QualityStats.cs
@@ -0,0 +1,339 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+
+namespace Lucene.Net.Benchmarks.Quality
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Results of quality benchmark run for a single query or for a set of queries.
+ /// </summary>
+ public class QualityStats
+ {
+ /// <summary>Number of points for which precision is computed.</summary>
+ public static readonly int MAX_POINTS = 20;
+
+ private double maxGoodPoints;
+ private double recall;
+ private double[] pAt;
+ private double pReleventSum = 0;
+ private double numPoints = 0;
+ private double numGoodPoints = 0;
+ private double mrr = 0;
+ private long searchTime;
+ private long docNamesExtractTime;
+
+ /// <summary>
+ /// A certain rank in which a relevant doc was found.
+ /// </summary>
+ public class RecallPoint
+ {
+ private int rank;
+ private double recall;
+ internal RecallPoint(int rank, double recall)
+ {
+ this.rank = rank;
+ this.recall = recall;
+ }
+
+ /// <summary>Returns the rank: where on the list of returned docs this relevant doc appeared.</summary>
+ public virtual int Rank
+ {
+ get { return rank; }
+ }
+
+ /// <summary>Returns the recall: how many relevant docs were returned up to this point, inclusive.</summary>
+ public virtual double Recall
+ {
+ get { return recall; }
+ }
+ }
+
+ private IList<RecallPoint> recallPoints;
+
+ /// <summary>
+ /// Construct a QualityStats object with anticipated maximal number of relevant hits.
+ /// </summary>
+ /// <param name="maxGoodPoints">maximal possible relevant hits.</param>
+ /// <param name="searchTime"></param>
+ public QualityStats(double maxGoodPoints, long searchTime)
+ {
+ this.maxGoodPoints = maxGoodPoints;
+ this.searchTime = searchTime;
+ this.recallPoints = new List<RecallPoint>();
+ pAt = new double[MAX_POINTS + 1]; // pAt[0] unused.
+ }
+
+ /// <summary>
+ /// Add a (possibly relevant) doc.
+ /// </summary>
+ /// <param name="n">rank of the added doc (its ordinal position within the query results).</param>
+ /// <param name="isRelevant"><c>true</c> if the added doc is relevant, <c>false</c> otherwise.</param>
+ /// <param name="docNameExtractTime"></param>
+ public virtual void AddResult(int n, bool isRelevant, long docNameExtractTime)
+ {
+ if (Math.Abs(numPoints + 1 - n) > 1E-6)
+ {
+ throw new ArgumentException("point " + n + " illegal after " + numPoints + " points!");
+ }
+ if (isRelevant)
+ {
+ numGoodPoints += 1;
+ recallPoints.Add(new RecallPoint(n, numGoodPoints));
+ if (recallPoints.Count == 1 && n <= 5)
+ { // first point, but only within 5 top scores.
+ mrr = 1.0 / n;
+ }
+ }
+ numPoints = n;
+ double p = numGoodPoints / numPoints;
+ if (isRelevant)
+ {
+ pReleventSum += p;
+ }
+ if (n < pAt.Length)
+ {
+ pAt[n] = p;
+ }
+ recall = maxGoodPoints <= 0 ? p : numGoodPoints / maxGoodPoints;
+ docNamesExtractTime += docNameExtractTime;
+ }
+
+ /// <summary>
+ /// Return the precision at rank n:
+ /// |{relevant hits within first <c>n</c> hits}| / <c>n</c>.
+ /// </summary>
+ /// <param name="n">requested precision point, must be at least 1 and at most <see cref="MAX_POINTS"/>.</param>
+ /// <returns></returns>
+ public virtual double GetPrecisionAt(int n)
+ {
+ if (n < 1 || n > MAX_POINTS)
+ {
+ throw new ArgumentException("n=" + n + " - but it must be in [1," + MAX_POINTS + "] range!");
+ }
+ if (n > numPoints)
+ {
+ return (numPoints * pAt[(int)numPoints]) / n;
+ }
+ return pAt[n];
+ }
+
+ /// <summary>
+ /// Return the average precision at recall points.
+ /// </summary>
+ public virtual double GetAvp()
+ {
+ return maxGoodPoints == 0 ? 0 : pReleventSum / maxGoodPoints;
+ }
+
+ /// <summary>
+ /// Return the recall: |{relevant hits found}| / |{relevant hits existing}|.
+ /// </summary>
+ public virtual double Recall
+ {
+ get { return recall; }
+ }
+
+ /// <summary>
+ /// Log information on this <see cref="QualityStats"/> object.
+ /// </summary>
+ /// <param name="title"></param>
+ /// <param name="paddLines"></param>
+ /// <param name="logger">Logger.</param>
+ /// <param name="prefix">prefix before each log line.</param>
+ public virtual void Log(string title, int paddLines, TextWriter logger, string prefix)
+ {
+ for (int i = 0; i < paddLines; i++)
+ {
+ logger.WriteLine();
+ }
+ if (title != null && title.Trim().Length > 0)
+ {
+ logger.WriteLine(title);
+ }
+ prefix = prefix == null ? "" : prefix;
+ string nf = "{0:F3}";
+ int M = 19;
+ logger.WriteLine(prefix + Format("Search Seconds: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, (double)searchTime / 1000)));
+ logger.WriteLine(prefix + Format("DocName Seconds: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, (double)docNamesExtractTime / 1000)));
+ logger.WriteLine(prefix + Format("Num Points: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, numPoints)));
+ logger.WriteLine(prefix + Format("Num Good Points: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, numGoodPoints)));
+ logger.WriteLine(prefix + Format("Max Good Points: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, maxGoodPoints)));
+ logger.WriteLine(prefix + Format("Average Precision: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, GetAvp())));
+ logger.WriteLine(prefix + Format("MRR: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, MRR)));
+ logger.WriteLine(prefix + Format("Recall: ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, Recall)));
+ for (int i = 1; i < (int)numPoints && i < pAt.Length; i++)
+ {
+ logger.WriteLine(prefix + Format("Precision At " + i + ": ", M) +
+ FracFormat(string.Format(CultureInfo.InvariantCulture, nf, GetPrecisionAt(i))));
+ }
+ for (int i = 0; i < paddLines; i++)
+ {
+ logger.WriteLine();
+ }
+ }
+
+ private static string padd = " ";
+ private string Format(string s, int minLen)
+ {
+ s = (s == null ? "" : s);
+ int n = Math.Max(minLen, s.Length);
+ return (s + padd).Substring(0, n-0);
+ }
+ private string FracFormat(string frac)
+ {
+ int k = frac.IndexOf('.');
+ string s1 = padd + frac.Substring(0, k-0);
+ int n = Math.Max(k, 6);
+ s1 = s1.Substring(s1.Length - n);
+ return s1 + frac.Substring(k);
+ }
+
+ /// <summary>
+ /// Create a <see cref="QualityStats"/> object that is the average of the input <see cref="QualityStats"/> objects.
+ /// </summary>
+ /// <param name="stats">array of input stats to be averaged.</param>
+ /// <returns>an average over the input stats.</returns>
+ public static QualityStats Average(QualityStats[] stats)
+ {
+ QualityStats avg = new QualityStats(0, 0);
+ if (stats.Length == 0)
+ {
+ // weired, no stats to average!
+ return avg;
+ }
+ int m = 0; // queries with positive judgements
+ // aggregate
+ for (int i = 0; i < stats.Length; i++)
+ {
+ avg.searchTime += stats[i].searchTime;
+ avg.docNamesExtractTime += stats[i].docNamesExtractTime;
+ if (stats[i].maxGoodPoints > 0)
+ {
+ m++;
+ avg.numGoodPoints += stats[i].numGoodPoints;
+ avg.numPoints += stats[i].numPoints;
+ avg.pReleventSum += stats[i].GetAvp();
+ avg.recall += stats[i].recall;
+ avg.mrr += stats[i].MRR;
+ avg.maxGoodPoints += stats[i].maxGoodPoints;
+ for (int j = 1; j < avg.pAt.Length; j++)
+ {
+ avg.pAt[j] += stats[i].GetPrecisionAt(j);
+ }
+ }
+ }
+ Debug.Assert(m> 0, "Fishy: no \"good\" queries!");
+ // take average: times go by all queries, other measures go by "good" queries only.
+ avg.searchTime /= stats.Length;
+ avg.docNamesExtractTime /= stats.Length;
+ avg.numGoodPoints /= m;
+ avg.numPoints /= m;
+ avg.recall /= m;
+ avg.mrr /= m;
+ avg.maxGoodPoints /= m;
+ for (int j = 1; j < avg.pAt.Length; j++)
+ {
+ avg.pAt[j] /= m;
+ }
+ avg.pReleventSum /= m; // this is actually avgp now
+ avg.pReleventSum *= avg.maxGoodPoints; // so that getAvgP() would be correct
+
+ return avg;
+ }
+
+ /// <summary>
+ /// Returns the time it took to extract doc names for judging the measured query, in milliseconds.
+ /// </summary>
+ public virtual long DocNamesExtractTime
+ {
+ get { return docNamesExtractTime; }
+ }
+
+ /// <summary>
+ /// Returns the maximal number of good points.
+ /// This is the number of relevant docs known by the judge for the measured query.
+ /// </summary>
+ public virtual double MaxGoodPoints
+ {
+ get { return maxGoodPoints; }
+ }
+
+ /// <summary>
+ /// Returns the number of good points (only relevant points).
+ /// </summary>
+ public virtual double NumGoodPoints
+ {
+ get { return numGoodPoints; }
+ }
+
+ /// <summary>
+ /// Returns the number of points (both relevant and irrelevant points).
+ /// </summary>
+ public virtual double NumPoints
+ {
+ get { return numPoints; }
+ }
+
+ /// <summary>
+ /// Returns the recallPoints.
+ /// </summary>
+ public virtual RecallPoint[] GetRecallPoints()
+ {
+ return recallPoints.ToArray();
+ }
+
+ /// <summary>
+ /// Returns the Mean reciprocal rank over the queries or RR for a single query.
+ /// </summary>
+ /// <remarks>
+ /// Reciprocal rank is defined as <c>1/r</c> where <c>r</c> is the
+ /// rank of the first correct result, or <c>0</c> if there are no correct
+ /// results within the top 5 results.
+ /// <para/>
+ /// This follows the definition in
+ /// <a href="http://www.cnlp.org/publications/02cnlptrec10.pdf">
+ /// Question Answering - CNLP at the TREC-10 Question Answering Track</a>.
+ /// </remarks>
+ public virtual double MRR
+ {
+ get { return mrr; }
+ }
+
+
+ /// <summary>
+ /// Returns the search time in milliseconds for the measured query.
+ /// </summary>
+ public virtual long SearchTime
+ {
+ get { return searchTime; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/Quality/Trec/QueryDriver.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Quality/Trec/QueryDriver.cs b/src/Lucene.Net.Benchmark/Quality/Trec/QueryDriver.cs
new file mode 100644
index 0000000..0540e62
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Quality/Trec/QueryDriver.cs
@@ -0,0 +1,93 @@
+using Lucene.Net.Benchmarks.Quality.Utils;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.Quality.Trec
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Command-line tool for doing a TREC evaluation run.
+ /// </summary>
+ public class QueryDriver
+ {
+ public static void Main(string[] args)
+ {
+ if (args.Length < 4 || args.Length > 5)
+ {
+ SystemConsole.Error.WriteLine("Usage: QueryDriver <topicsFile> <qrelsFile> <submissionFile> <indexDir> [querySpec]");
+ SystemConsole.Error.WriteLine("topicsFile: input file containing queries");
+ SystemConsole.Error.WriteLine("qrelsFile: input file containing relevance judgements");
+ SystemConsole.Error.WriteLine("submissionFile: output submission file for trec_eval");
+ SystemConsole.Error.WriteLine("indexDir: index directory");
+ SystemConsole.Error.WriteLine("querySpec: string composed of fields to use in query consisting of T=title,D=description,N=narrative:");
+ SystemConsole.Error.WriteLine("\texample: TD (query on Title + Description). The default is T (title only)");
+ Environment.Exit(1);
+ }
+
+ FileInfo topicsFile = new FileInfo(args[0]);
+ FileInfo qrelsFile = new FileInfo(args[1]);
+ SubmissionReport submitLog = new SubmissionReport(new StreamWriter(new FileStream(args[2], FileMode.Create, FileAccess.Write), Encoding.UTF8 /* huh, no nio.Charset ctor? */), "lucene");
+ using (Store.FSDirectory dir = Store.FSDirectory.Open(new DirectoryInfo(args[3])))
+ using (IndexReader reader = DirectoryReader.Open(dir))
+ {
+ string fieldSpec = args.Length == 5 ? args[4] : "T"; // default to Title-only if not specified.
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ int maxResults = 1000;
+ string docNameField = "docname";
+
+ TextWriter logger = SystemConsole.Out; //new StreamWriter(SystemConsole, Encoding.GetEncoding(0));
+
+ // use trec utilities to read trec topics into quality queries
+ TrecTopicsReader qReader = new TrecTopicsReader();
+ QualityQuery[] qqs = qReader.ReadQueries(IOUtils.GetDecodingReader(topicsFile, Encoding.UTF8));
+
+ // prepare judge, with trec utilities that read from a QRels file
+ IJudge judge = new TrecJudge(IOUtils.GetDecodingReader(qrelsFile, Encoding.UTF8));
+
+ // validate topics & judgments match each other
+ judge.ValidateData(qqs, logger);
+
+ ISet<string> fieldSet = new HashSet<string>();
+ if (fieldSpec.IndexOf('T') >= 0) fieldSet.Add("title");
+ if (fieldSpec.IndexOf('D') >= 0) fieldSet.Add("description");
+ if (fieldSpec.IndexOf('N') >= 0) fieldSet.Add("narrative");
+
+ // set the parsing of quality queries into Lucene queries.
+ IQualityQueryParser qqParser = new SimpleQQParser(fieldSet.ToArray(), "body");
+
+ // run the benchmark
+ QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
+ qrun.MaxResults = maxResults;
+ QualityStats[] stats = qrun.Execute(judge, submitLog, logger);
+
+ // print an avarage sum of the results
+ QualityStats avg = QualityStats.Average(stats);
+ avg.Log("SUMMARY", 2, logger, " ");
+ }
+ }
+ }
+}
[11/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
new file mode 100644
index 0000000..c9db5a2
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiContentSource.cs
@@ -0,0 +1,394 @@
+// LUCENENET TODO: Use HTML Agility pack instead of SAX ?
+
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support.Threading;
+using Lucene.Net.Util;
+using Sax.Net;
+using Sax.Net.Helpers;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="ContentSource"/> which reads the English Wikipedia dump. You can read
+ /// the <c>.bz2</c> file directly (it will be decompressed on the fly). Config
+ /// properties:
+ /// <list type="bullet">
+ /// <item><term>keep.image.only.docs</term><description>false|true (default <b>true</b>).</description></item>
+ /// <item><term>docs.file</term><description><path to the file></description></item>
+ /// </list>
+ /// </summary>
+ public class EnwikiContentSource : ContentSource
+ {
+ private class Parser : DefaultHandler//, IRunnable
+ {
+ private ThreadClass t;
+ private bool threadDone;
+ private bool stopped = false;
+ private string[] tuple;
+ private NoMoreDataException nmde;
+ private StringBuilder contents = new StringBuilder();
+ private string title;
+ private string body;
+ private string time;
+ private string id;
+
+ private readonly EnwikiContentSource outerInstance;
+
+ public Parser(EnwikiContentSource outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ internal string[] Next()
+ {
+ if (t == null)
+ {
+ threadDone = false;
+ t = new ThreadClass(/*this*/);
+ t.SetDaemon(true);
+ t.Start();
+ }
+ string[] result;
+ lock (this)
+ {
+ while (tuple == null && nmde == null && !threadDone && !stopped)
+ {
+ try
+ {
+ Monitor.Wait(this);
+ }
+ catch (ThreadInterruptedException ie)
+ {
+ throw new ThreadInterruptedException(ie.ToString(), ie);
+ }
+ }
+ if (tuple != null)
+ {
+ result = tuple;
+ tuple = null;
+ Monitor.Pulse(this);// notify();
+ return result;
+ }
+ if (nmde != null)
+ {
+ // Set to null so we will re-start thread in case
+ // we are re-used:
+ t = null;
+ throw nmde;
+ }
+ // The thread has exited yet did not hit end of
+ // data, so this means it hit an exception. We
+ // throw NoMorDataException here to force
+ // benchmark to stop the current alg:
+ throw new NoMoreDataException();
+ }
+ }
+
+ internal string Time(string original)
+ {
+ StringBuilder buffer = new StringBuilder();
+
+ buffer.Append(original.Substring(8, 10 - 8));
+ buffer.Append('-');
+ buffer.Append(months[Convert.ToInt32(original.Substring(5, 7 - 5), CultureInfo.InvariantCulture) - 1]);
+ buffer.Append('-');
+ buffer.Append(original.Substring(0, 4 - 0));
+ buffer.Append(' ');
+ buffer.Append(original.Substring(11, 19 - 11));
+ buffer.Append(".000");
+
+ return buffer.ToString();
+ }
+
+ public override void Characters(char[] ch, int start, int length)
+ {
+ contents.Append(ch, start, length);
+ }
+
+ public override void EndElement(string @namespace, string simple, string qualified)
+ {
+ int elemType = GetElementType(qualified);
+ switch (elemType)
+ {
+ case PAGE:
+ // the body must be null and we either are keeping image docs or the
+ // title does not start with Image:
+ if (body != null && (outerInstance.keepImages || !title.StartsWith("Image:", StringComparison.Ordinal)))
+ {
+ string[] tmpTuple = new string[LENGTH];
+ tmpTuple[TITLE] = title.Replace('\t', ' ');
+ tmpTuple[DATE] = time.Replace('\t', ' ');
+ tmpTuple[BODY] = Regex.Replace(body, "[\t\n]", " ");
+ tmpTuple[ID] = id;
+ lock (this)
+ {
+ while (tuple != null && !stopped)
+ {
+ try
+ {
+ Monitor.Wait(this); //wait();
+ }
+ catch (ThreadInterruptedException ie)
+ {
+ throw new ThreadInterruptedException(ie.ToString(), ie);
+ }
+ }
+ tuple = tmpTuple;
+ Monitor.Pulse(this); //notify();
+ }
+ }
+ break;
+ case BODY:
+ body = contents.ToString();
+ //workaround that startswith doesn't have an ignore case option, get at least 20 chars.
+ string startsWith = body.Substring(0, Math.Min(10, contents.Length)).ToLowerInvariant();
+ if (startsWith.StartsWith("#redirect", StringComparison.Ordinal))
+ {
+ body = null;
+ }
+ break;
+ case DATE:
+ time = Time(contents.ToString());
+ break;
+ case TITLE:
+ title = contents.ToString();
+ break;
+ case ID:
+ //the doc id is the first one in the page. All other ids after that one can be ignored according to the schema
+ if (id == null)
+ {
+ id = contents.ToString();
+ }
+ break;
+ default:
+ // this element should be discarded.
+ break;
+ }
+ }
+
+ public void Run()
+ {
+
+ try
+ {
+ Sax.Net.IXmlReader reader = XmlReaderFactory.Current.CreateXmlReader(); //XMLReaderFactory.createXMLReader();
+ reader.ContentHandler = this;
+ reader.ErrorHandler = this;
+ while (!stopped)
+ {
+ Stream localFileIS = outerInstance.@is;
+ if (localFileIS != null)
+ { // null means fileIS was closed on us
+ try
+ {
+ // To work around a bug in XERCES (XERCESJ-1257), we assume the XML is always UTF8, so we simply provide reader.
+ reader.Parse(new InputSource(IOUtils.GetDecodingReader(localFileIS, Encoding.UTF8)));
+ }
+ catch (IOException ioe)
+ {
+ lock (outerInstance)
+ {
+ if (localFileIS != outerInstance.@is)
+ {
+ // fileIS was closed on us, so, just fall through
+ }
+ else
+ // Exception is real
+ throw ioe;
+ }
+ }
+ }
+ lock (this)
+ {
+ if (stopped || !outerInstance.m_forever)
+ {
+ nmde = new NoMoreDataException();
+ Monitor.Pulse(this); //notify();
+ return;
+ }
+ else if (localFileIS == outerInstance.@is)
+ {
+ // If file is not already re-opened then re-open it now
+ outerInstance.@is = outerInstance.OpenInputStream();
+ }
+ }
+ }
+ }
+ catch (SAXException sae)
+ {
+ throw new Exception(sae.ToString(), sae);
+ }
+ catch (IOException ioe)
+ {
+ throw new Exception(ioe.ToString(), ioe);
+ }
+ finally
+ {
+ lock (this)
+ {
+ threadDone = true;
+ Monitor.Pulse(this); //Notify();
+ }
+ }
+ }
+
+ public override void StartElement(string @namespace, string simple, string qualified,
+ IAttributes attributes)
+ {
+ int elemType = GetElementType(qualified);
+ switch (elemType)
+ {
+ case PAGE:
+ title = null;
+ body = null;
+ time = null;
+ id = null;
+ break;
+ // intentional fall-through.
+ case BODY:
+ case DATE:
+ case TITLE:
+ case ID:
+ contents.Length = 0;
+ break;
+ default:
+ // this element should be discarded.
+ break;
+ }
+ }
+
+ internal void Stop()
+ {
+ lock (this)
+ {
+ stopped = true;
+ if (tuple != null)
+ {
+ tuple = null;
+ Monitor.Pulse(this); //Notify();
+ }
+ }
+ }
+
+ }
+
+ private static readonly IDictionary<string, int?> ELEMENTS = new Dictionary<string, int?>();
+ private const int TITLE = 0;
+ private const int DATE = TITLE + 1;
+ private const int BODY = DATE + 1;
+ private const int ID = BODY + 1;
+ private const int LENGTH = ID + 1;
+ // LENGTH is used as the size of the tuple, so whatever constants we need that
+ // should not be part of the tuple, we should define them after LENGTH.
+ private const int PAGE = LENGTH + 1;
+
+ private static readonly string[] months = {"JAN", "FEB", "MAR", "APR",
+ "MAY", "JUN", "JUL", "AUG",
+ "SEP", "OCT", "NOV", "DEC"};
+
+ static EnwikiContentSource()
+ {
+ ELEMENTS["page"] = PAGE;
+ ELEMENTS["text"] = BODY;
+ ELEMENTS["timestamp"] = DATE;
+ ELEMENTS["title"] = TITLE;
+ ELEMENTS["id"] = ID;
+ }
+
+ public EnwikiContentSource()
+ {
+ parser = new Parser(this);
+ }
+
+ /// <summary>
+ /// Returns the type of the element if defined, otherwise returns -1. This
+ /// method is useful in startElement and endElement, by not needing to compare
+ /// the element qualified name over and over.
+ /// </summary>
+ private static int GetElementType(string elem)
+ {
+ int? val;
+ ELEMENTS.TryGetValue(elem, out val);
+ return val == null ? -1 : val.Value;
+ }
+
+ private FileInfo file;
+ private bool keepImages = true;
+ private Stream @is;
+ private Parser parser;
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ lock (this)
+ {
+ parser.Stop();
+ if (@is != null)
+ {
+ @is.Dispose();
+ @is = null;
+ }
+ }
+ }
+ }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ string[] tuple = parser.Next();
+ docData.Clear();
+ docData.Name = tuple[ID];
+ docData.Body = tuple[BODY];
+ docData.SetDate(tuple[DATE]);
+ docData.Title = tuple[TITLE];
+ return docData;
+ }
+
+ public override void ResetInputs()
+ {
+ base.ResetInputs();
+ @is = OpenInputStream();
+ }
+
+ /// <summary>Open the input stream.</summary>
+ protected virtual Stream OpenInputStream()
+ {
+ return StreamUtils.GetInputStream(file);
+ }
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+ keepImages = config.Get("keep.image.only.docs", true);
+ string fileName = config.Get("docs.file", null);
+ if (fileName != null)
+ {
+ file = new FileInfo(fileName);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiQueryMaker.cs
new file mode 100644
index 0000000..77a86a1
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/EnwikiQueryMaker.cs
@@ -0,0 +1,146 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Index;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Spans;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A QueryMaker that uses common and uncommon actual Wikipedia queries for
+ /// searching the English Wikipedia collection. 90 queries total.
+ /// </summary>
+ public class EnwikiQueryMaker : AbstractQueryMaker, IQueryMaker
+ {
+ // common and a few uncommon queries from wikipedia search logs
+ private static string[] STANDARD_QUERIES = { "Images catbox gif",
+ "Imunisasi haram", "Favicon ico", "Michael jackson", "Unknown artist",
+ "Lily Thai", "Neda", "The Last Song", "Metallica", "Nicola Tesla",
+ "Max B", "Skil Corporation", "\"The 100 Greatest Artists of All Time\"",
+ "\"Top 100 Global Universities\"", "Pink floyd", "Bolton Sullivan",
+ "Frank Lucas Jr", "Drake Woods", "Radiohead", "George Freeman",
+ "Oksana Grigorieva", "The Elder Scrolls V", "Deadpool", "Green day",
+ "\"Red hot chili peppers\"", "Jennifer Bini Taylor",
+ "The Paradiso Girls", "Queen", "3Me4Ph", "Paloma Jimenez", "AUDI A4",
+ "Edith Bouvier Beale: A Life In Pictures", "\"Skylar James Deleon\"",
+ "Simple Explanation", "Juxtaposition", "The Woody Show", "London WITHER",
+ "In A Dark Place", "George Freeman", "LuAnn de Lesseps", "Muhammad.",
+ "U2", "List of countries by GDP", "Dean Martin Discography", "Web 3.0",
+ "List of American actors", "The Expendables",
+ "\"100 Greatest Guitarists of All Time\"", "Vince Offer.",
+ "\"List of ZIP Codes in the United States\"", "Blood type diet",
+ "Jennifer Gimenez", "List of hobbies", "The beatles", "Acdc",
+ "Nightwish", "Iron maiden", "Murder Was the Case", "Pelvic hernia",
+ "Naruto Shippuuden", "campaign", "Enthesopathy of hip region",
+ "operating system", "mouse",
+ "List of Xbox 360 games without region encoding", "Shakepearian sonnet",
+ "\"The Monday Night Miracle\"", "India", "Dad's Army",
+ "Solanum melanocerasum", "\"List of PlayStation Portable Wi-Fi games\"",
+ "Little Pixie Geldof", "Planes, Trains & Automobiles", "Freddy Ingalls",
+ "The Return of Chef", "Nehalem", "Turtle", "Calculus", "Superman-Prime",
+ "\"The Losers\"", "pen-pal", "Audio stream input output", "lifehouse",
+ "50 greatest gunners", "Polyfecalia", "freeloader", "The Filthy Youth" };
+
+ private static Query[] GetPrebuiltQueries(string field)
+ {
+ WildcardQuery wcq = new WildcardQuery(new Term(field, "fo*"));
+ wcq.MultiTermRewriteMethod = MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE;
+ // be wary of unanalyzed text
+ return new Query[] {
+ new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 5),
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanTermQuery(new Term(field, "night")),
+ new SpanTermQuery(new Term(field, "trading")) }, 4, false),
+ new SpanNearQuery(new SpanQuery[] {
+ new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 10),
+ new SpanTermQuery(new Term(field, "credit")) }, 10, false), wcq, };
+ }
+
+ /// <summary>
+ /// Parse the strings containing Lucene queries.
+ /// </summary>
+ /// <param name="qs">array of strings containing query expressions</param>
+ /// <param name="a">analyzer to use when parsing queries</param>
+ /// <returns>array of Lucene queries</returns>
+ private static Query[] CreateQueries(IList<object> qs, Analyzer a)
+ {
+ QueryParser qp = new QueryParser(
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT,
+#pragma warning restore 612, 618
+ DocMaker.BODY_FIELD, a);
+ IList<Query> queries = new List<Query>();
+ for (int i = 0; i < qs.Count; i++)
+ {
+ try
+ {
+
+ object query = qs[i];
+ Query q = null;
+ if (query is string)
+ {
+ q = qp.Parse((string)query);
+
+ }
+ else if (query is Query)
+ {
+ q = (Query)query;
+
+ }
+ else
+ {
+ SystemConsole.WriteLine("Unsupported Query Type: " + query);
+ }
+
+ if (q != null)
+ {
+ queries.Add(q);
+ }
+
+ }
+ catch (Exception e)
+ {
+ SystemConsole.WriteLine(e.ToString());
+ }
+ }
+
+ return queries.ToArray();
+ }
+
+ protected override Query[] PrepareQueries()
+ {
+ // analyzer (default is standard analyzer)
+ Analyzer anlzr = NewAnalyzerTask.CreateAnalyzer(m_config.Get("analyzer", typeof(StandardAnalyzer).AssemblyQualifiedName));
+
+ List<object> queryList = new List<object>(20);
+ queryList.AddRange(STANDARD_QUERIES);
+ if (!m_config.Get("enwikiQueryMaker.disableSpanQueries", false))
+ queryList.AddRange(GetPrebuiltQueries(DocMaker.BODY_FIELD));
+ return CreateQueries(queryList, anlzr);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/FacetSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/FacetSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/FacetSource.cs
new file mode 100644
index 0000000..51f0354
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/FacetSource.cs
@@ -0,0 +1,47 @@
+using Lucene.Net.Facet;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Source items for facets.
+ /// <para/>
+ /// For supported configuration parameters see <see cref="ContentItemsSource"/>.
+ /// </summary>
+ public abstract class FacetSource : ContentItemsSource
+ {
+ /// <summary>
+ /// Fills the next facets content items in the given list. Implementations must
+ /// account for multi-threading, as multiple threads can call this method
+ /// simultaneously.
+ /// </summary>
+ public abstract void GetNextFacets(IList<FacetField> facets);
+
+ public abstract void Configure(FacetsConfig config);
+
+ public override void ResetInputs()
+ {
+ PrintStatistics("facets");
+ // re-initiate since properties by round may have changed.
+ SetConfig(Config);
+ base.ResetInputs();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/FileBasedQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/FileBasedQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/FileBasedQueryMaker.cs
new file mode 100644
index 0000000..bf287e0
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/FileBasedQueryMaker.cs
@@ -0,0 +1,121 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Create queries from a <see cref="FileStream"/>. One per line, pass them through the
+ /// QueryParser. Lines beginning with # are treated as comments.
+ /// </summary>
+ /// <remarks>
+ /// File can be specified as a absolute, relative or resource.
+ /// Two properties can be set:
+ /// <list type="bullet">
+ /// <item><term>file.query.maker.file</term><description><Full path to file containing queries></description></item>
+ /// <item><term>file.query.maker.default.field</term><description><Name of default field - Default value is "body"></description></item>
+ /// </list>
+ /// <para/>
+ /// Example:
+ /// <code>
+ /// file.query.maker.file=c:/myqueries.txt
+ /// file.query.maker.default.field=body
+ /// </code>
+ /// </remarks>
+ public class FileBasedQueryMaker : AbstractQueryMaker, IQueryMaker
+ {
+ protected override Query[] PrepareQueries()
+ {
+ Analyzer anlzr = NewAnalyzerTask.CreateAnalyzer(m_config.Get("analyzer",
+ "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common"));
+ string defaultField = m_config.Get("file.query.maker.default.field", DocMaker.BODY_FIELD);
+ QueryParser qp = new QueryParser(
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT,
+#pragma warning restore 612, 618
+ defaultField, anlzr);
+ qp.AllowLeadingWildcard = true;
+
+ List<Query> qq = new List<Query>();
+ string fileName = m_config.Get("file.query.maker.file", null);
+ if (fileName != null)
+ {
+ FileInfo file = new FileInfo(fileName);
+ TextReader reader = null;
+ // note: we use a decoding reader, so if your queries are screwed up you know
+ if (file.Exists)
+ {
+ reader = IOUtils.GetDecodingReader(file, Encoding.UTF8);
+ }
+ else
+ {
+ //see if we can find it as a resource
+ Stream asStream = typeof(FileBasedQueryMaker).GetTypeInfo().Assembly.FindAndGetManifestResourceStream(typeof(FileBasedQueryMaker), fileName);
+ if (asStream != null)
+ {
+ reader = IOUtils.GetDecodingReader(asStream, Encoding.UTF8);
+ }
+ }
+ if (reader != null)
+ {
+ try
+ {
+ string line = null;
+ int lineNum = 0;
+ while ((line = reader.ReadLine()) != null)
+ {
+ line = line.Trim();
+ if (line.Length != 0 && !line.StartsWith("#", StringComparison.Ordinal))
+ {
+ try
+ {
+ qq.Add(qp.Parse(line));
+ }
+ catch (ParseException e)
+ {
+ SystemConsole.Error.WriteLine("Exception: " + e.Message + " occurred while parsing line: " + lineNum + " Text: " + line);
+ }
+ }
+ lineNum++;
+ }
+ }
+ finally
+ {
+ reader.Dispose();
+ }
+ }
+ else
+ {
+ SystemConsole.Error.WriteLine("No Reader available for: " + fileName);
+ }
+
+ }
+ return qq.ToArray();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/GeonamesLineParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/GeonamesLineParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/GeonamesLineParser.cs
new file mode 100644
index 0000000..d786a91
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/GeonamesLineParser.cs
@@ -0,0 +1,53 @@
+using System;
+using System.Text.RegularExpressions;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A line parser for Geonames.org data.
+ /// See <a href="http://download.geonames.org/export/dump/readme.txt">'geoname' table</a>.
+ /// Requires <see cref="SpatialDocMaker"/>.
+ /// </summary>
+ public class GeonamesLineParser : LineParser
+ {
+ /// <summary>
+ /// This header will be ignored; the geonames format is fixed and doesn't have a header line.
+ /// </summary>
+ public GeonamesLineParser(string[] header)
+ : base(header)
+ {
+ }
+
+ public override void ParseLine(DocData docData, string line)
+ {
+ string[] parts = new Regex("\\t").Split(line, 7);//no more than first 6 fields needed
+
+ // Sample data line:
+ // 3578267, Morne du Vitet, Morne du Vitet, 17.88333, -62.8, ...
+ // ID, Name, Alternate name (unused), Lat, Lon, ...
+
+ docData.ID = Convert.ToInt32(parts[0]);//note: overwrites ID assigned by LineDocSource
+ docData.Name = parts[1];
+ string latitude = parts[4];
+ string longitude = parts[5];
+ docData.Body = "POINT(" + longitude + " " + latitude + ")";//WKT is x y order
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/HTMLParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/HTMLParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/HTMLParser.cs
new file mode 100644
index 0000000..75e683b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/HTMLParser.cs
@@ -0,0 +1,42 @@
+using System;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// HTML Parsing Interface for test purposes.
+ /// </summary>
+ public interface IHTMLParser
+ {
+ /// <summary>
+ /// Parse the input TextReader and return DocData.
+ /// The provided name, title, date are used for the result, unless when they're null,
+ /// in which case an attempt is made to set them from the parsed data.
+ /// </summary>
+ /// <param name="docData">Result reused.</param>
+ /// <param name="name">Name of the result doc data.</param>
+ /// <param name="date">Date of the result doc data. If null, attempt to set by parsed data.</param>
+ /// <param name="reader">Reader of html text to parse.</param>
+ /// <param name="trecSrc">The <see cref="TrecContentSource"/> used to parse dates.</param>
+ /// <returns>Parsed doc data.</returns>
+ /// <exception cref="IOException">If there is a low-level I/O error.</exception>
+ DocData Parse(DocData docData, string name, DateTime? date, TextReader reader, TrecContentSource trecSrc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/LineDocSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/LineDocSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/LineDocSource.cs
new file mode 100644
index 0000000..1a45cc6
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/LineDocSource.cs
@@ -0,0 +1,328 @@
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="ContentSource"/> reading one line at a time as a
+ /// <see cref="Documents.Document"/> from a single file. This saves IO
+ /// cost (over DirContentSource) of recursing through a directory and opening a
+ /// new file for every document.
+ /// </summary>
+ /// <remarks>
+ /// The expected format of each line is (arguments are separated by <TAB>):
+ /// <i>title, date, body</i>. If a line is read in a different format, a
+ /// <see cref="Exception"/> will be thrown. In general, you should use this
+ /// content source for files that were created with <see cref="WriteLineDocTask"/>.
+ /// </remarks>
+ public class LineDocSource : ContentSource
+ {
+ // LUCENENET specific - de-nested LineParser, SimpleLineParser, HeaderLineParser
+
+ private FileInfo file;
+ private TextReader reader;
+ private int readCount;
+
+ private LineParser docDataLineReader = null;
+ private bool skipHeaderLine = false;
+
+ private void OpenFile()
+ {
+ try
+ {
+ if (reader != null)
+ {
+ reader.Dispose();
+ }
+ Stream @is = StreamUtils.GetInputStream(file);
+ reader = new StreamReader(@is, m_encoding);
+ if (skipHeaderLine)
+ {
+ reader.ReadLine(); // skip one line - the header line - already handled that info
+ }
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing && reader != null)
+ {
+ reader.Dispose();
+ reader = null;
+ }
+ }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ string line;
+ int myID;
+
+
+ lock (this)
+ {
+ line = reader.ReadLine();
+ if (line == null)
+ {
+ if (!m_forever)
+ {
+ throw new NoMoreDataException();
+ }
+ // Reset the file
+ OpenFile();
+ return GetNextDocData(docData);
+ }
+ if (docDataLineReader == null)
+ { // first line ever, one time initialization,
+ docDataLineReader = CreateDocDataLineReader(line);
+ if (skipHeaderLine)
+ {
+ return GetNextDocData(docData);
+ }
+ }
+ // increment IDS only once...
+ myID = readCount++;
+ }
+
+ // The date String was written in the format of DateTools.dateToString.
+ docData.Clear();
+ docData.ID = myID;
+ docDataLineReader.ParseLine(docData, line);
+ return docData;
+ }
+
+ private LineParser CreateDocDataLineReader(string line)
+ {
+ string[] header;
+ string headIndicator = WriteLineDocTask.FIELDS_HEADER_INDICATOR + WriteLineDocTask.SEP;
+
+ if (line.StartsWith(headIndicator, StringComparison.Ordinal))
+ {
+ header = line.Substring(headIndicator.Length).Split(new char[] { WriteLineDocTask.SEP }).TrimEnd();
+ skipHeaderLine = true; // mark to skip the header line when input file is reopened
+ }
+ else
+ {
+ header = WriteLineDocTask.DEFAULT_FIELDS;
+ }
+
+ // if a specific DocDataLineReader was configured, must respect it
+ string docDataLineReaderClassName = Config.Get("line.parser", null);
+ if (docDataLineReaderClassName != null)
+ {
+ try
+ {
+ Type clazz = Type.GetType(docDataLineReaderClassName);
+ return (LineParser)Activator.CreateInstance(clazz, (object)header);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("Failed to instantiate " + docDataLineReaderClassName, e);
+ }
+ }
+
+ // if this the simple case,
+ if (Arrays.Equals(header, WriteLineDocTask.DEFAULT_FIELDS))
+ {
+ return new SimpleLineParser(header);
+ }
+ return new HeaderLineParser(header);
+ }
+
+ public override void ResetInputs()
+ {
+ base.ResetInputs();
+ OpenFile();
+ }
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+ string fileName = config.Get("docs.file", null);
+ if (fileName == null)
+ {
+ throw new ArgumentException("docs.file must be set");
+ }
+ file = new FileInfo(fileName);
+ if (m_encoding == null)
+ {
+ m_encoding = Encoding.UTF8;
+ }
+ }
+ }
+
+ /// <summary>Reader of a single input line into <see cref="DocData"/>.</summary>
+ public abstract class LineParser
+ {
+ protected readonly string[] m_header;
+
+ /// <summary>
+ /// Construct with the header
+ /// </summary>
+ /// <param name="header">header line found in the input file, or <c>null</c> if none.</param>
+ public LineParser(string[] header)
+ {
+ this.m_header = header;
+ }
+
+ /// <summary>
+ /// parse an input line and fill doc data appropriately
+ /// </summary>
+ public abstract void ParseLine(DocData docData, string line);
+ }
+
+ /// <summary>
+ /// <see cref="LineParser"/> which ignores the header passed to its constructor
+ /// and assumes simply that field names and their order are the same
+ /// as in <see cref="WriteLineDocTask.DEFAULT_FIELDS"/>.
+ /// </summary>
+ public class SimpleLineParser : LineParser
+ {
+ public SimpleLineParser(string[] header)
+ : base(header)
+ {
+ }
+
+ public override void ParseLine(DocData docData, string line)
+ {
+ int k1 = 0;
+ int k2 = line.IndexOf(WriteLineDocTask.SEP, k1);
+ if (k2 < 0)
+ {
+ throw new Exception("line: [" + line + "] is in an invalid format (missing: separator title::date)!");
+ }
+ docData.Title = line.Substring(k1, k2 - k1);
+ k1 = k2 + 1;
+ k2 = line.IndexOf(WriteLineDocTask.SEP, k1);
+ if (k2 < 0)
+ {
+ throw new Exception("line: [" + line + "] is in an invalid format (missing: separator date::body)!");
+ }
+ docData.SetDate(line.Substring(k1, k2 - k1));
+ k1 = k2 + 1;
+ k2 = line.IndexOf(WriteLineDocTask.SEP, k1);
+ if (k2 >= 0)
+ {
+ throw new Exception("line: [" + line + "] is in an invalid format (too many separators)!");
+ }
+ // last one
+ docData.Body = line.Substring(k1);
+ }
+ }
+
+ /// <summary>
+ /// <see cref="LineParser"/> which sets field names and order by
+ /// the header - any header - of the lines file.
+ /// It is less efficient than <see cref="SimpleLineParser"/> but more powerful.
+ /// </summary>
+ public class HeaderLineParser : LineParser
+ {
+ private enum FieldName { NAME, TITLE, DATE, BODY, PROP }
+ private readonly FieldName[] posToF;
+ public HeaderLineParser(string[] header)
+ : base(header)
+ {
+ posToF = new FieldName[header.Length];
+ for (int i = 0; i < header.Length; i++)
+ {
+ String f = header[i];
+ if (DocMaker.NAME_FIELD.Equals(f, StringComparison.Ordinal))
+ {
+ posToF[i] = FieldName.NAME;
+ }
+ else if (DocMaker.TITLE_FIELD.Equals(f, StringComparison.Ordinal))
+ {
+ posToF[i] = FieldName.TITLE;
+ }
+ else if (DocMaker.DATE_FIELD.Equals(f, StringComparison.Ordinal))
+ {
+ posToF[i] = FieldName.DATE;
+ }
+ else if (DocMaker.BODY_FIELD.Equals(f, StringComparison.Ordinal))
+ {
+ posToF[i] = FieldName.BODY;
+ }
+ else
+ {
+ posToF[i] = FieldName.PROP;
+ }
+ }
+ }
+
+ public override void ParseLine(DocData docData, string line)
+ {
+ int n = 0;
+ int k1 = 0;
+ int k2;
+ while ((k2 = line.IndexOf(WriteLineDocTask.SEP, k1)) >= 0)
+ {
+ if (n >= m_header.Length)
+ {
+ throw new Exception("input line has invalid format: " + (n + 1) + " fields instead of " + m_header.Length + " :: [" + line + "]");
+ }
+ SetDocDataField(docData, n, line.Substring(k1, k2 - k1));
+ ++n;
+ k1 = k2 + 1;
+ }
+ if (n != m_header.Length - 1)
+ {
+ throw new Exception("input line has invalid format: " + (n + 1) + " fields instead of " + m_header.Length + " :: [" + line + "]");
+ }
+ // last one
+ SetDocDataField(docData, n, line.Substring(k1));
+ }
+
+ private void SetDocDataField(DocData docData, int position, string text)
+ {
+ switch (posToF[position])
+ {
+ case FieldName.NAME:
+ docData.Name = text;
+ break;
+ case FieldName.TITLE:
+ docData.Title = text;
+ break;
+ case FieldName.DATE:
+ docData.SetDate(text);
+ break;
+ case FieldName.BODY:
+ docData.Body = text;
+ break;
+ case FieldName.PROP:
+ var p = docData.Props;
+ if (p == null)
+ {
+ p = new Dictionary<string, string>();
+ docData.Props = p;
+ }
+ p[m_header[position]] = text;
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
new file mode 100644
index 0000000..fadab82
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishContentSource.cs
@@ -0,0 +1,72 @@
+using System;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Creates documents whose content is a <see cref="long"/> number starting from
+ /// <c><see cref="long.MinValue"/> + 10</c>.
+ /// </summary>
+ public class Int64ToEnglishContentSource : ContentSource
+ {
+ private long counter = 0;
+
+ protected override void Dispose(bool disposing)
+ {
+ }
+
+ // TODO: we could take param to specify locale...
+ //private readonly RuleBasedNumberFormat rnbf = new RuleBasedNumberFormat(Locale.ROOT,
+ // RuleBasedNumberFormat.SPELLOUT);
+ public override DocData GetNextDocData(DocData docData)
+ {
+ lock (this)
+ {
+ docData.Clear();
+ // store the current counter to avoid synchronization later on
+ long curCounter;
+ lock (this)
+ {
+ curCounter = counter;
+ if (counter == long.MaxValue)
+ {
+ counter = long.MinValue;//loop around
+ }
+ else
+ {
+ ++counter;
+ }
+ }
+
+ // LUCENENET TODO: Rules based number formatting...(from ICU)
+ docData.Body = curCounter.ToString(); //rnbf.format(curCounter);
+ docData.Name = "doc_" + curCounter.ToString(CultureInfo.InvariantCulture);
+ docData.Title = "title_" + curCounter.ToString(CultureInfo.InvariantCulture);
+ docData.SetDate(new DateTime());
+ return docData;
+ }
+ }
+
+ public override void ResetInputs()
+ {
+ counter = long.MinValue + 10;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
new file mode 100644
index 0000000..f565eb8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/LongToEnglishQueryMaker.cs
@@ -0,0 +1,89 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Util;
+using System;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Creates queries whose content is a spelled-out <see cref="long"/> number
+ /// starting from <c><see cref="long.MinValue"/> + 10</c>.
+ /// </summary>
+ public class Int64ToEnglishQueryMaker : IQueryMaker
+ {
+ long counter = long.MinValue + 10;
+ protected QueryParser m_parser;
+
+ //// TODO: we could take param to specify locale...
+ //private readonly RuleBasedNumberFormat rnbf = new RuleBasedNumberFormat(Locale.ROOT,
+ // RuleBasedNumberFormat.SPELLOUT);
+
+ public virtual Query MakeQuery(int size)
+ {
+ throw new NotSupportedException();
+ }
+
+ public virtual Query MakeQuery()
+ {
+ lock (this)
+ {
+ // LUCENENET TODO: Rules based number formatter (from ICU)
+ //return parser.Parse("" + rnbf.format(GetNextCounter()) + "");
+ return m_parser.Parse(GetNextCounter().ToString());
+ }
+ }
+
+ private long GetNextCounter()
+ {
+ lock (this)
+ {
+ if (counter == long.MaxValue)
+ {
+ counter = long.MinValue + 10;
+ }
+ return counter++;
+ }
+ }
+
+ public virtual void SetConfig(Config config)
+ {
+ Analyzer anlzr = NewAnalyzerTask.CreateAnalyzer(config.Get("analyzer", typeof(StandardAnalyzer).Name));
+ m_parser = new QueryParser(
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT,
+#pragma warning restore 612, 618
+ DocMaker.BODY_FIELD, anlzr);
+ }
+
+ public virtual void ResetInputs()
+ {
+ counter = long.MinValue + 10;
+ }
+
+ public virtual string PrintQueries()
+ {
+ return "LongToEnglish: [" + long.MinValue + " TO " + counter + "]";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/NoMoreDataException.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/NoMoreDataException.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/NoMoreDataException.cs
new file mode 100644
index 0000000..a7bfbad
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/NoMoreDataException.cs
@@ -0,0 +1,50 @@
+using System;
+#if FEATURE_SERIALIZABLE
+using System.Runtime.Serialization;
+#endif
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Exception indicating there is no more data.
+ /// Thrown by Docs Makers if <c>doc.maker.forever</c> is <c>false</c> and docs sources of that maker where exhausted.
+ /// This is useful for iterating all document of a source, in case we don't know in advance how many docs there are.
+ /// </summary>
+#if FEATURE_SERIALIZABLE
+ [Serializable]
+#endif
+ public class NoMoreDataException : Exception
+ {
+ public NoMoreDataException()
+ { }
+
+#if FEATURE_SERIALIZABLE
+ /// <summary>
+ /// Initializes a new instance of this class with serialized data.
+ /// </summary>
+ /// <param name="info">The <see cref="SerializationInfo"/> that holds the serialized object data about the exception being thrown.</param>
+ /// <param name="context">The <see cref="StreamingContext"/> that contains contextual information about the source or destination.</param>
+ public NoMoreDataException(SerializationInfo info, StreamingContext context)
+ : base(info, context)
+ {
+ }
+#endif
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/QueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/QueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/QueryMaker.cs
new file mode 100644
index 0000000..2ece812
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/QueryMaker.cs
@@ -0,0 +1,48 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Search;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Create queries for the test.
+ /// </summary>
+ public interface IQueryMaker
+ {
+ /// <summary>
+ /// Create the next query, of the given size.
+ /// </summary>
+ /// <param name="size">The size of the query - number of terms, etc.</param>
+ /// <returns></returns>
+ /// <exception cref="Exception">If cannot make the query, or if size > 0 was specified but this feature is not supported.</exception>
+ Query MakeQuery(int size);
+
+ /// <summary>Create the next query</summary>
+ Query MakeQuery();
+
+ /// <summary>Set the properties</summary>
+ void SetConfig(Config config);
+
+ /// <summary>Reset inputs so that the test run would behave, input wise, as if it just started.</summary>
+ void ResetInputs();
+
+ /// <summary>Print the queries</summary>
+ string PrintQueries();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/RandomFacetSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/RandomFacetSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/RandomFacetSource.cs
new file mode 100644
index 0000000..b956570
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/RandomFacetSource.cs
@@ -0,0 +1,109 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Facet;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simple implementation of a random facet source.
+ /// </summary>
+ /// <remarks>
+ /// Supports the following parameters:
+ /// <list type="bullet">
+ /// <item><term>rand.seed</term><description>defines the seed to initialize <see cref="Random"/> with (default: <b>13</b>).</description></item>
+ /// <item><term>max.doc.facet.dims</term><description>Max number of random dimensions to create (default: <b>5</b>);
+ /// actual number of dimensions would be anything between 1 and that number.</description></item>
+ /// <item><term>max.doc.facets</term><description>maximal #facets per doc (default: <b>10</b>).
+ /// Actual number of facets in a certain doc would be anything between 1 and that number.
+ /// </description></item>
+ /// <item><term>max.facet.depth</term><description>maximal #components in a facet (default:
+ /// <b>3</b>). Actual number of components in a certain facet would be anything
+ /// between 1 and that number.
+ /// </description></item>
+ /// </list>
+ /// </remarks>
+ public class RandomFacetSource : FacetSource
+ {
+ private Random random;
+ private int maxDocFacets;
+ private int maxFacetDepth;
+ private int maxDims;
+ private int maxValue; // = maxDocFacets * maxFacetDepth;
+
+ public override void GetNextFacets(IList<FacetField> facets)
+ {
+ facets.Clear();
+ int numFacets = 1 + random.Next(maxDocFacets); // at least one facet to each doc
+ for (int i = 0; i < numFacets; i++)
+ {
+ int depth;
+ if (maxFacetDepth == 2)
+ {
+ depth = 2;
+ }
+ else
+ {
+ depth = 2 + random.Next(maxFacetDepth - 2); // depth < 2 is not useful
+ }
+
+ string dim = random.Next(maxDims).ToString(CultureInfo.InvariantCulture);
+ string[] components = new string[depth - 1];
+ for (int k = 0; k < depth - 1; k++)
+ {
+ components[k] = random.Next(maxValue).ToString(CultureInfo.InvariantCulture);
+ AddItem();
+ }
+ FacetField ff = new FacetField(dim, components);
+ facets.Add(ff);
+ AddBytes(ff.ToString().Length); // very rough approximation
+ }
+ }
+
+ public override void Configure(FacetsConfig config)
+ {
+ for (int i = 0; i < maxDims; i++)
+ {
+ config.SetHierarchical(i.ToString(CultureInfo.InvariantCulture), true);
+ config.SetMultiValued(i.ToString(CultureInfo.InvariantCulture), true);
+ }
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ // nothing to do here
+ }
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+ random = new Random(config.Get("rand.seed", 13));
+ maxDocFacets = config.Get("max.doc.facets", 10);
+ maxDims = config.Get("max.doc.facets.dims", 5);
+ maxFacetDepth = config.Get("max.facet.depth", 3);
+ if (maxFacetDepth < 2)
+ {
+ throw new ArgumentException("max.facet.depth must be at least 2; got: " + maxFacetDepth);
+ }
+ maxValue = maxDocFacets * maxFacetDepth;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersContentSource.cs
new file mode 100644
index 0000000..c61ce2f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersContentSource.cs
@@ -0,0 +1,140 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="ContentSource"/> reading from the Reuters collection.
+ /// <para/>
+ /// Config properties:
+ /// <list type="bullet">
+ /// <item><term><b>work.dir</b></term><description>path to the root of docs and indexes dirs (default <b>work</b>).</description></item>
+ /// <item><term><b>docs.dir</b></term><description>path to the docs dir (default <b>reuters-out</b>).</description></item>
+ /// </list>
+ /// </summary>
+ public class ReutersContentSource : ContentSource
+ {
+ // LUCENENET specific: DateFormatInfo not used
+
+ private DirectoryInfo dataDir = null;
+ private List<FileInfo> inputFiles = new List<FileInfo>();
+ private int nextFile = 0;
+ private int iteration = 0;
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+ DirectoryInfo workDir = new DirectoryInfo(config.Get("work.dir", "work"));
+ string d = config.Get("docs.dir", "reuters-out");
+ dataDir = new DirectoryInfo(d);
+ inputFiles.Clear();
+ CollectFiles(dataDir, inputFiles);
+ if (inputFiles.Count == 0)
+ {
+ throw new Exception("No txt files in dataDir: " + dataDir.FullName);
+ }
+ }
+
+ // LUCENENET specific: DateFormatInfo not used
+
+ private DateTime? ParseDate(string dateStr)
+ {
+ DateTime temp;
+ if (DateTime.TryParseExact(dateStr, "dd-MMM-yyyy hh:mm:ss.fff", CultureInfo.InvariantCulture, DateTimeStyles.None, out temp))
+ {
+ return temp;
+ }
+ else if (DateTime.TryParse(dateStr, CultureInfo.InvariantCulture, DateTimeStyles.None, out temp))
+ {
+ return temp;
+ }
+
+ return null;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ // TODO implement?
+ }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ FileInfo f = null;
+ string name = null;
+ lock (this)
+ {
+ if (nextFile >= inputFiles.Count)
+ {
+ // exhausted files, start a new round, unless forever set to false.
+ if (!m_forever)
+ {
+ throw new NoMoreDataException();
+ }
+ nextFile = 0;
+ iteration++;
+ }
+ f = inputFiles[nextFile++];
+ name = f.FullName + "_" + iteration;
+ }
+
+ using (TextReader reader = new StreamReader(new FileStream(f.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8))
+ {
+ // First line is the date, 3rd is the title, rest is body
+ string dateStr = reader.ReadLine();
+ reader.ReadLine();// skip an empty line
+ string title = reader.ReadLine();
+ reader.ReadLine();// skip an empty line
+ StringBuilder bodyBuf = new StringBuilder(1024);
+ string line = null;
+ while ((line = reader.ReadLine()) != null)
+ {
+ bodyBuf.Append(line).Append(' ');
+ }
+ reader.Dispose();
+
+
+ AddBytes(f.Length);
+
+ DateTime? date = ParseDate(dateStr.Trim());
+
+ docData.Clear();
+ docData.Name = name;
+ docData.Body = bodyBuf.ToString();
+ docData.Title = title;
+ docData.SetDate(date);
+ return docData;
+ }
+ }
+
+ public override void ResetInputs()
+ {
+ lock (this)
+ {
+ base.ResetInputs();
+ nextFile = 0;
+ iteration = 0;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersQueryMaker.cs
new file mode 100644
index 0000000..a53a2ec
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/ReutersQueryMaker.cs
@@ -0,0 +1,126 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Index;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Spans;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="IQueryMaker"/> that makes queries devised manually (by Grant Ingersoll) for
+ /// searching in the Reuters collection.
+ /// </summary>
+ public class ReutersQueryMaker : AbstractQueryMaker, IQueryMaker
+ {
+ private static string[] STANDARD_QUERIES = {
+ //Start with some short queries
+ "Salomon", "Comex", "night trading", "Japan Sony",
+ //Try some Phrase Queries
+ "\"Sony Japan\"", "\"food needs\"~3",
+ "\"World Bank\"^2 AND Nigeria", "\"World Bank\" -Nigeria",
+ "\"Ford Credit\"~5",
+ //Try some longer queries
+ "airline Europe Canada destination",
+ "Long term pressure by trade " +
+ "ministers is necessary if the current Uruguay round of talks on " +
+ "the General Agreement on Trade and Tariffs (GATT) is to " +
+ "succeed"
+ };
+
+ private static Query[] GetPrebuiltQueries(string field)
+ {
+ // be wary of unanalyzed text
+ return new Query[] {
+ new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 5),
+ new SpanNearQuery(new SpanQuery[]{new SpanTermQuery(new Term(field, "night")), new SpanTermQuery(new Term(field, "trading"))}, 4, false),
+ new SpanNearQuery(new SpanQuery[]{new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 10), new SpanTermQuery(new Term(field, "credit"))}, 10, false),
+ new WildcardQuery(new Term(field, "fo*")),
+ };
+ }
+
+ /// <summary>
+ /// Parse the strings containing Lucene queries.
+ /// </summary>
+ /// <param name="qs">array of strings containing query expressions</param>
+ /// <param name="a">analyzer to use when parsing queries</param>
+ /// <returns>array of Lucene queries</returns>
+ private static Query[] CreateQueries(IList<object> qs, Analyzer a)
+ {
+ QueryParser qp = new QueryParser(
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT,
+#pragma warning restore 612, 618
+ DocMaker.BODY_FIELD, a);
+ List<Query> queries = new List<Query>();
+ for (int i = 0; i < qs.Count; i++)
+ {
+ try
+ {
+
+ object query = qs[i];
+ Query q = null;
+ if (query is string)
+ {
+ q = qp.Parse((string)query);
+
+ }
+ else if (query is Query)
+ {
+ q = (Query)query;
+
+ }
+ else
+ {
+ SystemConsole.Error.WriteLine("Unsupported Query Type: " + query);
+ }
+
+ if (q != null)
+ {
+ queries.Add(q);
+ }
+
+ }
+ catch (Exception e)
+ {
+ SystemConsole.Error.WriteLine(e.ToString());
+ }
+ }
+
+ return queries.ToArray();
+ }
+
+ protected override Query[] PrepareQueries()
+ {
+ // analyzer (default is standard analyzer)
+ Analyzer anlzr = NewAnalyzerTask.CreateAnalyzer(m_config.Get("analyzer",
+ "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common"));
+
+ List<object> queryList = new List<object>(20);
+ queryList.AddRange(STANDARD_QUERIES);
+ queryList.AddRange(GetPrebuiltQueries(DocMaker.BODY_FIELD));
+ return CreateQueries(queryList, anlzr);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleQueryMaker.cs
new file mode 100644
index 0000000..4ba2fe3
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleQueryMaker.cs
@@ -0,0 +1,70 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Index;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Util;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="IQueryMaker"/> that makes queries for a collection created
+ /// using <see cref="SingleDocSource"/>.
+ /// </summary>
+ public class SimpleQueryMaker : AbstractQueryMaker, IQueryMaker
+ {
+ /// <summary>
+ /// Prepare the queries for this test.
+ /// Extending classes can override this method for preparing different queries.
+ /// </summary>
+ /// <returns>Prepared queries.</returns>
+ /// <exception cref="System.Exception">If cannot prepare the queries.</exception>
+ protected override Query[] PrepareQueries()
+ {
+ // analyzer (default is standard analyzer)
+ Analyzer anlzr = NewAnalyzerTask.CreateAnalyzer(m_config.Get("analyzer",
+ "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common"));
+
+ QueryParser qp = new QueryParser(
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT,
+#pragma warning restore 612, 618
+ DocMaker.BODY_FIELD, anlzr);
+ List<Query> qq = new List<Query>();
+ Query q1 = new TermQuery(new Term(DocMaker.ID_FIELD, "doc2"));
+ qq.Add(q1);
+ Query q2 = new TermQuery(new Term(DocMaker.BODY_FIELD, "simple"));
+ qq.Add(q2);
+ BooleanQuery bq = new BooleanQuery();
+ bq.Add(q1, Occur.MUST);
+ bq.Add(q2, Occur.MUST);
+ qq.Add(bq);
+ qq.Add(qp.Parse("synthetic body"));
+ qq.Add(qp.Parse("\"synthetic body\""));
+ qq.Add(qp.Parse("synthetic text"));
+ qq.Add(qp.Parse("\"synthetic text\""));
+ qq.Add(qp.Parse("\"synthetic text\"~3"));
+ qq.Add(qp.Parse("zoom*"));
+ qq.Add(qp.Parse("synth*"));
+ return qq.ToArray();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleSloppyPhraseQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleSloppyPhraseQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleSloppyPhraseQueryMaker.cs
new file mode 100644
index 0000000..7208b25
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/SimpleSloppyPhraseQueryMaker.cs
@@ -0,0 +1,88 @@
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Create sloppy phrase queries for performance test, in an index created using simple doc maker.
+ /// </summary>
+ public class SimpleSloppyPhraseQueryMaker : SimpleQueryMaker
+ {
+ /// <seealso cref="SimpleQueryMaker.PrepareQueries()"/>
+ protected override Query[] PrepareQueries()
+ {
+ // extract some 100 words from doc text to an array
+ string[] words;
+ List<string> w = new List<string>();
+ StringTokenizer st = new StringTokenizer(SingleDocSource.DOC_TEXT);
+ while (st.HasMoreTokens() && w.Count < 100)
+ {
+ w.Add(st.NextToken());
+ }
+ words = w.ToArray();
+
+ // create queries (that would find stuff) with varying slops
+ IList<Query> queries = new List<Query>();
+ for (int slop = 0; slop < 8; slop++)
+ {
+ for (int qlen = 2; qlen < 6; qlen++)
+ {
+ for (int wd = 0; wd < words.Length - qlen - slop; wd++)
+ {
+ // ordered
+ int remainedSlop = slop;
+ PhraseQuery q = new PhraseQuery();
+ q.Slop = slop;
+ int wind = wd;
+ for (int i = 0; i < qlen; i++)
+ {
+ q.Add(new Term(DocMaker.BODY_FIELD, words[wind++]));
+ if (remainedSlop > 0)
+ {
+ remainedSlop--;
+ wind++;
+ }
+ }
+ queries.Add(q);
+ // reversed
+ remainedSlop = slop;
+ q = new PhraseQuery();
+ q.Slop = slop + 2 * qlen;
+ wind = wd + qlen + remainedSlop - 1;
+ for (int i = 0; i < qlen; i++)
+ {
+ q.Add(new Term(DocMaker.BODY_FIELD, words[wind--]));
+ if (remainedSlop > 0)
+ {
+ remainedSlop--;
+ wind--;
+ }
+ }
+ queries.Add(q);
+ }
+ }
+ }
+ return queries.ToArray();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/SingleDocSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/SingleDocSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/SingleDocSource.cs
new file mode 100644
index 0000000..6b01faf
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/SingleDocSource.cs
@@ -0,0 +1,77 @@
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Creates the same document each time <see cref="GetNextDocData(DocData)"/> is called.
+ /// </summary>
+ public class SingleDocSource : ContentSource
+ {
+ private int docID = 0;
+
+ internal static readonly string DOC_TEXT =
+ "Well, this is just some plain text we use for creating the " +
+ "test documents. It used to be a text from an online collection " +
+ "devoted to first aid, but if there was there an (online) lawyers " +
+ "first aid collection with legal advices, \"it\" might have quite " +
+ "probably advised one not to include \"it\"'s text or the text of " +
+ "any other online collection in one's code, unless one has money " +
+ "that one don't need and one is happy to donate for lawyers " +
+ "charity. Anyhow at some point, rechecking the usage of this text, " +
+ "it became uncertain that this text is free to use, because " +
+ "the web site in the disclaimer of he eBook containing that text " +
+ "was not responding anymore, and at the same time, in projGut, " +
+ "searching for first aid no longer found that eBook as well. " +
+ "So here we are, with a perhaps much less interesting " +
+ "text for the test, but oh much much safer. ";
+
+ // return a new docid
+ private int NewDocID()
+ {
+ lock (this)
+ {
+ if (docID > 0 && !m_forever)
+ {
+ throw new NoMoreDataException();
+ }
+ return docID++;
+ }
+ }
+
+ protected override void Dispose(bool disposing) { }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ int id = NewDocID();
+ AddBytes(DOC_TEXT.Length);
+ docData.Clear();
+ docData.Name = "doc" + id;
+ docData.Body = DOC_TEXT;
+ return docData;
+ }
+
+ public override void ResetInputs()
+ {
+ lock (this)
+ {
+ base.ResetInputs();
+ docID = 0;
+ }
+ }
+ }
+}
[19/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.tt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.tt b/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.tt
new file mode 100644
index 0000000..5f4a839
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLSchema.tt
@@ -0,0 +1,72 @@
+<#@ template debug="true" hostspecific="true" language="C#" #>
+<#@ assembly name="System.Xml" #>
+<#@ import namespace="System.IO" #>
+<#@ import namespace="System.Xml.Xsl" #>
+<#@ output extension=".Generated.cs" #>
+<# /*
+# -----------------------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the ""License""); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ""AS IS"" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -----------------------------------------------------------------------------------
+*/ #>
+//------------------------------------------------------------------------------
+// <auto-generated>
+// This code was generated by a tool.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+namespace TagSoup
+{
+ /// <summary>
+ /// This class provides a Schema that has been preinitialized with HTML
+ /// elements, attributes, and character entity declarations. All the declarations
+ /// normally provided with HTML 4.01 are given, plus some that are IE-specific
+ /// and NS4-specific. Attribute declarations of type CDATA with no default
+ /// value are not included.
+ /// </summary>
+ public class HTMLSchema : Schema
+ {
+ // HTMLModels begin
+ <#
+ XslCompiledTransform transform = new XslCompiledTransform(true);
+ transform.Load(this.Host.ResolvePath("tssl/tssl-models.xslt"));
+ using(StringWriter writer = new StringWriter())
+ {
+ transform.Transform(this.Host.ResolvePath("definitions/html.tssl"), null, writer);
+ Write(writer.ToString());
+ }
+ #> // HTMLModels end
+
+ /// <summary>
+ /// Returns a newly constructed HTMLSchema object independent of
+ /// any existing ones.
+ /// </summary>
+ public HTMLSchema()
+ {
+ <#
+ transform.Load(this.Host.ResolvePath("tssl/tssl.xslt"));
+ using(StringWriter writer = new StringWriter())
+ {
+ transform.Transform(this.Host.ResolvePath("definitions/html.tssl"), null, writer);
+ Write(writer.ToString());
+ }
+ #>
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/PYXScanner.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/PYXScanner.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/PYXScanner.cs
new file mode 100644
index 0000000..711d46a
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/PYXScanner.cs
@@ -0,0 +1,138 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// This file is part of TagSoup.
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version. You may also distribute
+// and/or modify it under version 2.1 of the Academic Free License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// PYX Scanner
+
+using System.IO;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// A <see cref="IScanner"/> that accepts PYX format instead of HTML.
+ /// Useful primarily for debugging.
+ /// </summary>
+ public class PYXScanner : IScanner
+ {
+ public virtual void ResetDocumentLocator(string publicid, string systemid)
+ {
+ // Need this method for interface compatibility, but note
+ // that PyxScanner does not implement Locator.
+ }
+
+ public virtual void Scan(TextReader br, IScanHandler h)
+ {
+ string s;
+ char[] buff = null;
+ bool instag = false;
+ while ((s = br.ReadLine()) != null)
+ {
+ int size = s.Length;
+ buff = s.ToCharArray(0, size);
+ if (buff.Length < size)
+ {
+ buff = new char[size];
+ }
+ switch (buff[0])
+ {
+ case '(':
+ if (instag)
+ {
+ h.STagC(buff, 0, 0);
+ instag = false;
+ }
+ h.GI(buff, 1, size - 1);
+ instag = true;
+ break;
+ case ')':
+ if (instag)
+ {
+ h.STagC(buff, 0, 0);
+ instag = false;
+ }
+ h.ETag(buff, 1, size - 1);
+ break;
+ case '?':
+ if (instag)
+ {
+ h.STagC(buff, 0, 0);
+ instag = false;
+ }
+ h.PI(buff, 1, size - 1);
+ break;
+ case 'A':
+ int sp = s.IndexOf(' ');
+ h.Aname(buff, 1, sp - 1);
+ h.Aval(buff, sp + 1, size - sp - 1);
+ break;
+ case '-':
+ if (instag)
+ {
+ h.STagC(buff, 0, 0);
+ instag = false;
+ }
+ if (s.Equals("-\\n"))
+ {
+ buff[0] = '\n';
+ h.PCDATA(buff, 0, 1);
+ }
+ else
+ {
+ // FIXME:
+ // Does not decode \t and \\ in input
+ h.PCDATA(buff, 1, size - 1);
+ }
+ break;
+ case 'E':
+ if (instag)
+ {
+ h.STagC(buff, 0, 0);
+ instag = false;
+ }
+ h.Entity(buff, 1, size - 1);
+ break;
+ default:
+ // System.err.print("Gotcha ");
+ // System.err.print(s);
+ // System.err.print('\n');
+ break;
+ }
+ }
+ h.EOF(buff, 0, 0);
+ }
+
+ public void StartCDATA()
+ {
+ }
+
+ //public static void main(string[] argv) {
+ // IScanner s = new PYXScanner();
+ // TextReader r = new StreamReader(System.Console.OpenStandardInput(), Encoding.UTF8);
+ // TextWriter w = new StreamWriter(System.Console.OpenStandardOutput(), Encoding.UTF8));
+ // s.Scan(r, new PYXWriter(w));
+ // }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
new file mode 100644
index 0000000..ff47d0d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/PYXWriter.cs
@@ -0,0 +1,286 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// PYX Writer
+// FIXME: does not do escapes in attribute values
+// FIXME: outputs entities as bare '&' character
+
+using Sax;
+using Sax.Ext;
+using System.IO;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// A <see cref="IContentHandler"/> that generates PYX format instead of XML.
+ /// Primarily useful for debugging.
+ /// </summary>
+ public class PYXWriter : IScanHandler, IContentHandler, ILexicalHandler
+ {
+ private readonly TextWriter theWriter; // where we Write to
+ private static char[] dummy = new char[1];
+ private string attrName; // saved attribute name
+
+ // ScanHandler implementation
+
+ public void Adup(char[] buff, int offset, int length)
+ {
+ theWriter.WriteLine(attrName);
+ attrName = null;
+ }
+
+ public void Aname(char[] buff, int offset, int length)
+ {
+ theWriter.Write('A');
+ theWriter.Write(buff, offset, length);
+ theWriter.Write(' ');
+ attrName = new string(buff, offset, length);
+ }
+
+ public void Aval(char[] buff, int offset, int length)
+ {
+ theWriter.Write(buff, offset, length);
+ theWriter.WriteLine();
+ attrName = null;
+ }
+
+ public void Cmnt(char[] buff, int offset, int length)
+ {
+ // theWriter.Write('!');
+ // theWriter.Write(buff, offset, length);
+ // theWriter.WriteLine();
+ }
+
+ public void Entity(char[] buff, int offset, int length)
+ {
+ }
+
+ public int GetEntity()
+ {
+ return 0;
+ }
+
+ public void EOF(char[] buff, int offset, int length)
+ {
+ theWriter.Close();
+ }
+
+ public void ETag(char[] buff, int offset, int length)
+ {
+ theWriter.Write(')');
+ theWriter.Write(buff, offset, length);
+ theWriter.WriteLine();
+ }
+
+ public void Decl(char[] buff, int offset, int length)
+ {
+ }
+
+ public void GI(char[] buff, int offset, int length)
+ {
+ theWriter.Write('(');
+ theWriter.Write(buff, offset, length);
+ theWriter.WriteLine();
+ }
+
+ public void CDSect(char[] buff, int offset, int length)
+ {
+ PCDATA(buff, offset, length);
+ }
+
+ public void PCDATA(char[] buff, int offset, int length)
+ {
+ if (length == 0)
+ {
+ return; // nothing to do
+ }
+ bool inProgress = false;
+ length += offset;
+ for (int i = offset; i < length; i++)
+ {
+ if (buff[i] == '\n')
+ {
+ if (inProgress)
+ {
+ theWriter.WriteLine();
+ }
+ theWriter.WriteLine("-\\n");
+ inProgress = false;
+ }
+ else
+ {
+ if (!inProgress)
+ {
+ theWriter.Write('-');
+ }
+ switch (buff[i])
+ {
+ case '\t':
+ theWriter.Write("\\t");
+ break;
+ case '\\':
+ theWriter.Write("\\\\");
+ break;
+ default:
+ theWriter.Write(buff[i]);
+ break;
+ }
+ inProgress = true;
+ }
+ }
+ if (inProgress)
+ {
+ theWriter.WriteLine();
+ }
+ }
+
+ public void PITarget(char[] buff, int offset, int length)
+ {
+ theWriter.Write('?');
+ theWriter.Write(buff, offset, length);
+ theWriter.Write(' ');
+ }
+
+ public void PI(char[] buff, int offset, int length)
+ {
+ theWriter.Write(buff, offset, length);
+ theWriter.WriteLine();
+ }
+
+ public void STagC(char[] buff, int offset, int length)
+ {
+ // theWriter.WriteLine("!"); // FIXME
+ }
+
+ public void STagE(char[] buff, int offset, int length)
+ {
+ theWriter.WriteLine("!"); // FIXME
+ }
+
+ // SAX ContentHandler implementation
+
+ public void Characters(char[] buff, int offset, int length)
+ {
+ PCDATA(buff, offset, length);
+ }
+
+ public void EndDocument()
+ {
+ theWriter.Close();
+ }
+
+ public void EndElement(string uri, string localname, string qname)
+ {
+ if (qname.Length == 0)
+ {
+ qname = localname;
+ }
+ theWriter.Write(')');
+ theWriter.WriteLine(qname);
+ }
+
+ public void EndPrefixMapping(string prefix)
+ {
+ }
+
+ public void IgnorableWhitespace(char[] buff, int offset, int length)
+ {
+ Characters(buff, offset, length);
+ }
+
+ public void ProcessingInstruction(string target, string data)
+ {
+ theWriter.Write('?');
+ theWriter.Write(target);
+ theWriter.Write(' ');
+ theWriter.WriteLine(data);
+ }
+
+ public void SetDocumentLocator(ILocator locator)
+ {
+ }
+
+ public void SkippedEntity(string name)
+ {
+ }
+
+ public void StartDocument()
+ {
+ }
+
+ public void StartElement(string uri, string localname, string qname, IAttributes atts)
+ {
+ if (qname.Length == 0)
+ {
+ qname = localname;
+ }
+ theWriter.Write('(');
+ theWriter.WriteLine(qname);
+ int length = atts.Length;
+ for (int i = 0; i < length; i++)
+ {
+ qname = atts.GetQName(i);
+ if (qname.Length == 0)
+ {
+ qname = atts.GetLocalName(i);
+ }
+ theWriter.Write('A');
+ // theWriter.Write(atts.getType(i)); // DEBUG
+ theWriter.Write(qname);
+ theWriter.Write(' ');
+ theWriter.WriteLine(atts.GetValue(i));
+ }
+ }
+
+ public void StartPrefixMapping(string prefix, string uri)
+ {
+ }
+
+ public void Comment(char[] ch, int start, int length)
+ {
+ Cmnt(ch, start, length);
+ }
+
+ public void EndCDATA()
+ {
+ }
+
+ public void EndDTD()
+ {
+ }
+
+ public void EndEntity(string name)
+ {
+ }
+
+ public void StartCDATA()
+ {
+ }
+
+ public void StartDTD(string name, string publicId, string systemId)
+ {
+ }
+
+ public void StartEntity(string name)
+ {
+ }
+
+ // Constructor
+
+ public PYXWriter(TextWriter w)
+ {
+ theWriter = w;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
new file mode 100644
index 0000000..a0a5463
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/Parser.cs
@@ -0,0 +1,1484 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// The TagSoup parser
+
+using Lucene.Net.Support;
+using Sax;
+using Sax.Ext;
+using Sax.Helpers;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// The SAX parser class.
+ /// </summary>
+ public class Parser : DefaultHandler, IScanHandler, IXMLReader, ILexicalHandler
+ {
+ // XMLReader implementation
+
+ private IContentHandler theContentHandler;
+ private ILexicalHandler theLexicalHandler;
+ private IDTDHandler theDTDHandler;
+ private IErrorHandler theErrorHandler;
+ private IEntityResolver theEntityResolver;
+ private Schema theSchema;
+ private IScanner theScanner;
+ private IAutoDetector theAutoDetector;
+
+ // Default values for feature flags
+
+ private const bool DEFAULT_NAMESPACES = true;
+ private const bool DEFAULT_IGNORE_BOGONS = false;
+ private const bool DEFAULT_BOGONS_EMPTY = false;
+ private const bool DEFAULT_ROOT_BOGONS = true;
+ private const bool DEFAULT_DEFAULT_ATTRIBUTES = true;
+ private const bool DEFAULT_TRANSLATE_COLONS = false;
+ private const bool DEFAULT_RESTART_ELEMENTS = true;
+ private const bool DEFAULT_IGNORABLE_WHITESPACE = false;
+ private const bool DEFAULT_CDATA_ELEMENTS = true;
+
+ // Feature flags.
+
+ private bool namespaces = DEFAULT_NAMESPACES;
+ private bool ignoreBogons = DEFAULT_IGNORE_BOGONS;
+ private bool bogonsEmpty = DEFAULT_BOGONS_EMPTY;
+ private bool rootBogons = DEFAULT_ROOT_BOGONS;
+ private bool defaultAttributes = DEFAULT_DEFAULT_ATTRIBUTES;
+ private bool translateColons = DEFAULT_TRANSLATE_COLONS;
+ private bool restartElements = DEFAULT_RESTART_ELEMENTS;
+ private bool ignorableWhitespace = DEFAULT_IGNORABLE_WHITESPACE;
+ private bool cDataElements = DEFAULT_CDATA_ELEMENTS;
+
+ /// <summary>
+ /// A value of "true" indicates namespace URIs and unprefixed local
+ /// names for element and attribute names will be available.
+ /// </summary>
+ public const string NAMESPACES_FEATURE = "http://xml.org/sax/features/namespaces";
+
+ /// <summary>
+ /// A value of "true" indicates that XML qualified names (with prefixes)
+ /// and attributes (including xmlns* attributes) will be available.
+ /// We don't support this value.
+ /// </summary>
+ public const string NAMESPACE_PREFIXES_FEATURE = "http://xml.org/sax/features/namespace-prefixes";
+
+ /// <summary>
+ /// Reports whether this parser processes external general entities
+ /// (it doe
+ /// </summary>
+ public const string EXTERNAL_GENERAL_ENTITIES_FEATURE = "http://xml.org/sax/features/external-general-entities";
+
+ /// <summary>
+ /// Reports whether this parser processes external parameter entities
+ /// (it doesn't).
+ /// </summary>
+ public const string EXTERNAL_PARAMETER_ENTITIES_FEATURE = "http://xml.org/sax/features/external-parameter-entities";
+
+ /// <summary>
+ /// May be examined only during a parse, after the startDocument()
+ /// callback has been completed; read-only. The value is true if
+ /// the document specified standalone="yes" in its XML declaration,
+ /// and otherwise is false. (It's always false.)
+ /// </summary>
+ public const string IS_STANDALONE_FEATURE = "http://xml.org/sax/features/is-standalone";
+
+ /// <summary>
+ /// A value of "true" indicates that the LexicalHandler will report
+ /// the beginning and end of parameter entities (it won't).
+ /// </summary>
+ public const string LEXICAL_HANDLER_PARAMETER_ENTITIES_FEATURE =
+ "http://xml.org/sax/features/lexical-handler/parameter-entities";
+
+ /// <summary>
+ /// A value of "true" indicates that system IDs in declarations will
+ /// be absolutized (relative to their base URIs) before reporting.
+ /// (This returns true but doesn't actually do anything.)
+ /// </summary>
+ public const string RESOLVE_DTD_URIS_FEATURE = "http://xml.org/sax/features/resolve-dtd-uris";
+
+ /// <summary>
+ /// Has a value of "true" if all XML names (for elements,
+ /// prefixes, attributes, entities, notations, and local
+ /// names), as well as Namespace URIs, will have been interned
+ /// using <see cref="string.Intern" />. This supports fast testing of
+ /// equality/inequality against string constants, rather than forcing
+ /// slower calls to <see cref="string.Equals(object)" />. (We always intern.)
+ /// </summary>
+ public const string STRING_INTERNING_FEATURE = "http://xml.org/sax/features/string-interning";
+
+ /// <summary>
+ /// Returns "true" if the Attributes objects passed by this
+ /// parser in <see cref="IContentHandler.StartElement" /> implement the
+ /// <see cref="Sax.Net.Ext.IAttributes2" /> interface. (They don't.)
+ /// </summary>
+ public const string USE_ATTRIBUTES2_FEATURE = "http://xml.org/sax/features/use-attributes2";
+
+ /// <summary>
+ /// Returns "true" if the Locator objects passed by this parser
+ /// parser in <see cref="IContentHandler.SetDocumentLocator" /> implement the
+ /// <see cref="Sax.Net.Ext.ILocator2" /> interface. (They don't.)
+ /// </summary>
+ public const string USE_LOCATOR2_FEATURE = "http://xml.org/sax/features/use-locator2";
+ /// <summary>
+ /// Returns "true" if, when setEntityResolver is given an object
+ /// implementing the <see cref="Sax.Net.Ext.IEntityResolver2" /> interface,
+ /// those new methods will be used. (They won't be.)
+ /// </summary>
+ public const string USE_ENTITY_RESOLVER2_FEATURE = "http://xml.org/sax/features/use-entity-resolver2";
+
+ /// <summary>
+ /// Controls whether the parser is reporting all validity errors
+ /// (We don't report any validity errors.)
+ /// </summary>
+ public const string VALIDATION_FEATURE = "http://xml.org/sax/features/validation";
+
+ /// <summary>
+ /// Controls whether the parser reports Unicode normalization
+ /// errors as described in section 2.13 and Appendix B of the XML
+ /// 1.1 Recommendation. (We don't normalize.)
+ /// </summary>
+ public const string UNICODE_NORMALIZATION_CHECKING_FEATURE =
+ "http://xml.org/sax/features/unicode-normalization-checking";
+
+ /// <summary>
+ /// Controls whether, when the namespace-prefixes feature is set,
+ /// the parser treats namespace declaration attributes as being in
+ /// the http://www.w3.org/2000/xmlns/ namespace. (It doesn't.)
+ /// </summary>
+ public const string XMLNS_URIS_FEATURE = "http://xml.org/sax/features/xmlns-uris";
+
+ /// <summary>
+ /// Returns <c>true</c> if the parser supports both XML 1.1 and XML 1.0.
+ /// (Always <c>false</c>.)
+ /// </summary>
+ public const string XML11_FEATURE = "http://xml.org/sax/features/xml-1.1";
+
+ /// <summary>
+ /// A value of <c>true</c> indicates that the parser will ignore
+ /// unknown elements.
+ /// </summary>
+ public const string IGNORE_BOGONS_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/ignore-bogons";
+
+ /// <summary>
+ /// A value of <c>true</c> indicates that the parser will give unknown
+ /// elements a content model of EMPTY; a value of <c>false</c>, a
+ /// content model of ANY.
+ /// </summary>
+ public const string BOGONS_EMPTY_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/bogons-empty";
+
+ /// <summary>
+ /// A value of <c>true</c> indicates that the parser will allow unknown
+ /// elements to be the root element.
+ /// </summary>
+ public const string ROOT_BOGONS_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/root-bogons";
+
+ /// <summary>
+ /// A value of <c>true</c> indicates that the parser will return default
+ /// attribute values for missing attributes that have default values.
+ /// </summary>
+ public const string DEFAULT_ATTRIBUTES_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/default-attributes";
+
+ /// <summary>
+ /// A value of <c>true</c> indicates that the parser will
+ /// translate colons into underscores in names.
+ /// </summary>
+ public const string TRANSLATE_COLONS_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/translate-colons";
+
+ /// <summary>
+ /// A value of <c>true</c> indicates that the parser will
+ /// attempt to restart the restartable elements.
+ /// </summary>
+ public const string RESTART_ELEMENTS_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/restart-elements";
+
+ /// <summary>
+ /// A value of "true" indicates that the parser will
+ /// transmit whitespace in element-only content via the SAX
+ /// ignorableWhitespace callback. Normally this is not done,
+ /// because HTML is an SGML application and SGML suppresses
+ /// such whitespace.
+ /// </summary>
+ public const string IGNORABLE_WHITESPACE_FEATURE =
+ "http://www.ccil.org/~cowan/tagsoup/features/ignorable-whitespace";
+
+ /// <summary>
+ /// A value of "true" indicates that the parser will treat CDATA
+ /// elements specially. Normally true, since the input is by
+ /// default HTML.
+ /// </summary>
+ public const string CDATA_ELEMENTS_FEATURE = "http://www.ccil.org/~cowan/tagsoup/features/cdata-elements";
+
+ /// <summary>
+ /// Used to see some syntax events that are essential in some
+ /// applications: comments, CDATA delimiters, selected general
+ /// entity inclusions, and the start and end of the DTD (and
+ /// declaration of document element name). The Object must implement
+ /// <see cref="ILexicalHandler" />
+ /// </summary>
+ public const string LEXICAL_HANDLER_PROPERTY = "http://xml.org/sax/properties/lexical-handler";
+
+ /// <summary>
+ /// Specifies the Scanner object this Parser uses.
+ /// </summary>
+ public const string SCANNER_PROPERTY = "http://www.ccil.org/~cowan/tagsoup/properties/scanner";
+
+ /// <summary>
+ /// Specifies the Schema object this Parser uses.
+ /// </summary>
+ public const string SCHEMA_PROPERTY = "http://www.ccil.org/~cowan/tagsoup/properties/schema";
+
+ /// <summary>
+ /// Specifies the AutoDetector (for encoding detection) this Parser uses.
+ /// </summary>
+ public const string AUTO_DETECTOR_PROPERTY = "http://www.ccil.org/~cowan/tagsoup/properties/auto-detector";
+
+
+ // Due to sucky Java order of initialization issues, these
+ // entries are maintained separately from the initial values of
+ // the corresponding instance variables, but care must be taken
+ // to keep them in sync.
+
+ private readonly Hashtable features = new Hashtable {
+ { NAMESPACES_FEATURE, DEFAULT_NAMESPACES },
+ { NAMESPACE_PREFIXES_FEATURE, false },
+ { EXTERNAL_GENERAL_ENTITIES_FEATURE, false },
+ { EXTERNAL_PARAMETER_ENTITIES_FEATURE, false },
+ { IS_STANDALONE_FEATURE, false },
+ { LEXICAL_HANDLER_PARAMETER_ENTITIES_FEATURE, false },
+ { RESOLVE_DTD_URIS_FEATURE, true },
+ { STRING_INTERNING_FEATURE, true },
+ { USE_ATTRIBUTES2_FEATURE, false },
+ { USE_LOCATOR2_FEATURE, false },
+ { USE_ENTITY_RESOLVER2_FEATURE, false },
+ { VALIDATION_FEATURE, false },
+ { XMLNS_URIS_FEATURE, false },
+ { XML11_FEATURE, false },
+ { IGNORE_BOGONS_FEATURE, DEFAULT_IGNORE_BOGONS },
+ { BOGONS_EMPTY_FEATURE, DEFAULT_BOGONS_EMPTY },
+ { ROOT_BOGONS_FEATURE, DEFAULT_ROOT_BOGONS },
+ { DEFAULT_ATTRIBUTES_FEATURE, DEFAULT_DEFAULT_ATTRIBUTES },
+ { TRANSLATE_COLONS_FEATURE, DEFAULT_TRANSLATE_COLONS },
+ { RESTART_ELEMENTS_FEATURE, DEFAULT_RESTART_ELEMENTS },
+ { IGNORABLE_WHITESPACE_FEATURE, DEFAULT_IGNORABLE_WHITESPACE },
+ { CDATA_ELEMENTS_FEATURE, DEFAULT_CDATA_ELEMENTS },
+ };
+
+ public virtual bool GetFeature(string name)
+ {
+ if (features.ContainsKey(name))
+ {
+ return (bool)features[name];
+ }
+ throw new SAXNotRecognizedException("Unknown feature " + name);
+ }
+
+ public virtual void SetFeature(string name, bool value)
+ {
+ if (false == features.ContainsKey(name))
+ {
+ throw new SAXNotRecognizedException("Unknown feature " + name);
+ }
+ features[name] = value;
+
+ if (name.Equals(NAMESPACES_FEATURE))
+ {
+ namespaces = value;
+ }
+ else if (name.Equals(IGNORE_BOGONS_FEATURE))
+ {
+ ignoreBogons = value;
+ }
+ else if (name.Equals(BOGONS_EMPTY_FEATURE))
+ {
+ bogonsEmpty = value;
+ }
+ else if (name.Equals(ROOT_BOGONS_FEATURE))
+ {
+ rootBogons = value;
+ }
+ else if (name.Equals(DEFAULT_ATTRIBUTES_FEATURE))
+ {
+ defaultAttributes = value;
+ }
+ else if (name.Equals(TRANSLATE_COLONS_FEATURE))
+ {
+ translateColons = value;
+ }
+ else if (name.Equals(RESTART_ELEMENTS_FEATURE))
+ {
+ restartElements = value;
+ }
+ else if (name.Equals(IGNORABLE_WHITESPACE_FEATURE))
+ {
+ ignorableWhitespace = value;
+ }
+ else if (name.Equals(CDATA_ELEMENTS_FEATURE))
+ {
+ cDataElements = value;
+ }
+ }
+
+ public virtual object GetProperty(string name)
+ {
+ if (name.Equals(LEXICAL_HANDLER_PROPERTY))
+ {
+ return theLexicalHandler == this ? null : theLexicalHandler;
+ }
+ if (name.Equals(SCANNER_PROPERTY))
+ {
+ return theScanner;
+ }
+ if (name.Equals(SCHEMA_PROPERTY))
+ {
+ return theSchema;
+ }
+ if (name.Equals(AUTO_DETECTOR_PROPERTY))
+ {
+ return theAutoDetector;
+ }
+ throw new SAXNotRecognizedException("Unknown property " + name);
+ }
+
+ public virtual void SetProperty(string name, object value)
+ {
+ if (name.Equals(LEXICAL_HANDLER_PROPERTY))
+ {
+ if (value == null)
+ {
+ theLexicalHandler = this;
+ }
+ else
+ {
+ var handler = value as ILexicalHandler;
+ if (handler != null)
+ {
+ theLexicalHandler = handler;
+ }
+ else
+ {
+ throw new SAXNotSupportedException("Your lexical handler is not a ILexicalHandler");
+ }
+ }
+ }
+ else if (name.Equals(SCANNER_PROPERTY))
+ {
+ var scanner = value as IScanner;
+ if (scanner != null)
+ {
+ theScanner = scanner;
+ }
+ else
+ {
+ throw new SAXNotSupportedException("Your scanner is not a IScanner");
+ }
+ }
+ else if (name.Equals(SCHEMA_PROPERTY))
+ {
+ var schema = value as Schema;
+ if (schema != null)
+ {
+ theSchema = schema;
+ }
+ else
+ {
+ throw new SAXNotSupportedException("Your schema is not a Schema");
+ }
+ }
+ else if (name.Equals(AUTO_DETECTOR_PROPERTY))
+ {
+ var detector = value as IAutoDetector;
+ if (detector != null)
+ {
+ theAutoDetector = detector;
+ }
+ else
+ {
+ throw new SAXNotSupportedException("Your auto-detector is not an IAutoDetector");
+ }
+ }
+ else
+ {
+ throw new SAXNotRecognizedException("Unknown property " + name);
+ }
+ }
+
+ public virtual IEntityResolver EntityResolver
+ {
+ get { return theEntityResolver == this ? null : theEntityResolver; }
+ set { theEntityResolver = value ?? this; }
+ }
+
+ public virtual IDTDHandler DTDHandler
+ {
+ get { return theDTDHandler == this ? null : theDTDHandler; }
+ set { theDTDHandler = value ?? this; }
+ }
+
+ public virtual IContentHandler ContentHandler
+ {
+ get { return theContentHandler == this ? null : theContentHandler; }
+ set { theContentHandler = value ?? this; }
+ }
+
+ public virtual IErrorHandler ErrorHandler
+ {
+ get { return theErrorHandler == this ? null : theErrorHandler; }
+ set { theErrorHandler = value ?? this; }
+ }
+
+ public virtual void Parse(InputSource input)
+ {
+ Setup();
+ TextReader r = GetReader(input);
+ theContentHandler.StartDocument();
+ theScanner.ResetDocumentLocator(input.PublicId, input.SystemId);
+ var locator = theScanner as ILocator;
+ if (locator != null)
+ {
+ theContentHandler.SetDocumentLocator(locator);
+ }
+ if (!(theSchema.Uri.Equals("")))
+ {
+ theContentHandler.StartPrefixMapping(theSchema.Prefix, theSchema.Uri);
+ }
+ theScanner.Scan(r, this);
+ }
+
+ public virtual void Parse(string systemid)
+ {
+ Parse(new InputSource(systemid));
+ }
+
+ // Sets up instance variables that haven't been set by setFeature
+ private void Setup()
+ {
+ if (theSchema == null)
+ {
+ theSchema = new HTMLSchema();
+ }
+ if (theScanner == null)
+ {
+ theScanner = new HTMLScanner();
+ }
+ if (theAutoDetector == null)
+ {
+ theAutoDetector = new AutoDetectorDelegate(stream => new StreamReader(stream));
+ }
+ theStack = new Element(theSchema.GetElementType("<root>"), defaultAttributes);
+ thePCDATA = new Element(theSchema.GetElementType("<pcdata>"), defaultAttributes);
+ theNewElement = null;
+ theAttributeName = null;
+ thePITarget = null;
+ theSaved = null;
+ theEntity = 0;
+ virginStack = true;
+ theDoctypeName = theDoctypePublicId = theDoctypeSystemId = null;
+ }
+
+ /// <summary>
+ /// Return a <see cref="TextReader"/> based on the contents of an <see cref="InputSource"/>
+ /// Buffer the Stream
+ /// </summary>
+ /// <param name="s"></param>
+ /// <returns></returns>
+ private TextReader GetReader(InputSource s)
+ {
+ TextReader r = s.TextReader;
+ Stream i = s.Stream;
+ Encoding encoding = s.Encoding;
+ string publicid = s.PublicId;
+ string systemid = s.SystemId;
+ if (r == null)
+ {
+ if (i == null)
+ {
+ i = GetInputStream(publicid, systemid);
+ }
+ if (!(i is BufferedStream))
+ {
+ i = new BufferedStream(i);
+ }
+ if (encoding == null)
+ {
+ r = theAutoDetector.AutoDetectingReader(i);
+ }
+ else
+ {
+ //try {
+ //TODO: Safe?
+ r = new StreamReader(i, encoding);
+ // }
+ //catch (UnsupportedEncodingException e) {
+ // r = new StreamReader(i);
+ // }
+ }
+ }
+ // r = new BufferedReader(r);
+ return r;
+ }
+
+ /// <summary>
+ /// Get an Stream based on a publicid and a systemid
+ /// We don't process publicids (who uses them anyhow?)
+ /// </summary>
+ /// <param name="publicid"></param>
+ /// <param name="systemid"></param>
+ /// <returns></returns>
+ private Stream GetInputStream(string publicid, string systemid)
+ {
+ var basis = new Uri("file://" + Environment.CurrentDirectory + Path.DirectorySeparatorChar);
+ var url = new Uri(basis, systemid);
+ return new FileStream(url.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+ }
+
+ // ScanHandler implementation
+
+ private Element theNewElement;
+ private string theAttributeName;
+ private bool theDoctypeIsPresent;
+ private string theDoctypePublicId;
+ private string theDoctypeSystemId;
+ private string theDoctypeName;
+ private string thePITarget;
+ private Element theStack;
+ private Element theSaved;
+ private Element thePCDATA;
+ private int theEntity; // needs to support chars past U+FFFF
+
+
+ public virtual void Adup(char[] buff, int offset, int length)
+ {
+ if (theNewElement == null || theAttributeName == null)
+ {
+ return;
+ }
+ theNewElement.SetAttribute(theAttributeName, null, theAttributeName);
+ theAttributeName = null;
+ }
+
+ public virtual void Aname(char[] buff, int offset, int length)
+ {
+ if (theNewElement == null)
+ {
+ return;
+ }
+ // Currently we don't rely on Schema to canonicalize
+ // attribute names.
+ theAttributeName = MakeName(buff, offset, length).ToLowerInvariant();
+ // System.err.println("%% Attribute name " + theAttributeName);
+ }
+
+ public virtual void Aval(char[] buff, int offset, int length)
+ {
+ if (theNewElement == null || theAttributeName == null)
+ {
+ return;
+ }
+ var value = new string(buff, offset, length);
+ // System.err.println("%% Attribute value [" + value + "]");
+ value = ExpandEntities(value);
+ theNewElement.SetAttribute(theAttributeName, null, value);
+ theAttributeName = null;
+ // System.err.println("%% Aval done");
+ }
+
+ /// <summary>
+ /// Expand entity references in attribute values selectively.
+ /// Currently we expand a reference iff it is properly terminated
+ /// with a semicolon.
+ /// </summary>
+ /// <param name="src"></param>
+ /// <returns></returns>
+ private string ExpandEntities(string src)
+ {
+ int refStart = -1;
+ int len = src.Length;
+ var dst = new char[len];
+ int dstlen = 0;
+ for (int i = 0; i < len; i++)
+ {
+ char ch = src[i];
+ dst[dstlen++] = ch;
+ // System.err.print("i = " + i + ", d = " + dstlen + ", ch = [" + ch + "] ");
+ if (ch == '&' && refStart == -1)
+ {
+ // start of a ref excluding &
+ refStart = dstlen;
+ // System.err.println("start of ref");
+ }
+ else if (refStart == -1)
+ {
+ // not in a ref
+ // System.err.println("not in ref");
+ }
+ else if (char.IsLetter(ch) || char.IsDigit(ch) || ch == '#')
+ {
+ // valid entity char
+ // System.err.println("valid");
+ }
+ else if (ch == ';')
+ {
+ // properly terminated ref
+ // System.err.print("got [" + new string(dst, refStart, dstlen-refStart-1) + "]");
+ int ent = LookupEntity(dst, refStart, dstlen - refStart - 1);
+ // System.err.println(" = " + ent);
+ if (ent > 0xFFFF)
+ {
+ ent -= 0x10000;
+ dst[refStart - 1] = (char)((ent >> 10) + 0xD800);
+ dst[refStart] = (char)((ent & 0x3FF) + 0xDC00);
+ dstlen = refStart + 1;
+ }
+ else if (ent != 0)
+ {
+ dst[refStart - 1] = (char)ent;
+ dstlen = refStart;
+ }
+ refStart = -1;
+ }
+ else
+ {
+ // improperly terminated ref
+ // System.err.println("end of ref");
+ refStart = -1;
+ }
+ }
+ return new string(dst, 0, dstlen);
+ }
+
+ public virtual void Entity(char[] buff, int offset, int length)
+ {
+ theEntity = LookupEntity(buff, offset, length);
+ }
+
+ /// <summary>
+ /// Process numeric character references,
+ /// deferring to the schema for named ones.
+ /// </summary>
+ /// <param name="buff"></param>
+ /// <param name="offset"></param>
+ /// <param name="length"></param>
+ /// <returns></returns>
+ private int LookupEntity(char[] buff, int offset, int length)
+ {
+ int result = 0;
+ if (length < 1)
+ {
+ return result;
+ }
+ // System.err.println("%% Entity at " + offset + " " + length);
+ // System.err.println("%% Got entity [" + new string(buff, offset, length) + "]");
+ if (buff[offset] == '#')
+ {
+ if (length > 1 && (buff[offset + 1] == 'x' || buff[offset + 1] == 'X'))
+ {
+ try
+ {
+ return Convert.ToInt32(new string(buff, offset + 2, length - 2), 16);
+ }
+ catch (FormatException)
+ {
+ return 0;
+ }
+ }
+ try
+ {
+ return Convert.ToInt32(new string(buff, offset + 1, length - 1), 10);
+ }
+ catch (FormatException)
+ {
+ return 0;
+ }
+ }
+ return theSchema.GetEntity(new string(buff, offset, length));
+ }
+
+ public virtual void EOF(char[] buff, int offset, int length)
+ {
+ if (virginStack)
+ {
+ Rectify(thePCDATA);
+ }
+ while (theStack.Next != null)
+ {
+ Pop();
+ }
+ if (!(theSchema.Uri.Equals("")))
+ {
+ theContentHandler.EndPrefixMapping(theSchema.Prefix);
+ }
+ theContentHandler.EndDocument();
+ }
+
+ public virtual void ETag(char[] buff, int offset, int length)
+ {
+ if (ETagCdata(buff, offset, length))
+ {
+ return;
+ }
+ ETagBasic(buff, offset, length);
+ }
+
+ private static readonly char[] etagchars = { '<', '/', '>' };
+ public virtual bool ETagCdata(char[] buff, int offset, int length)
+ {
+ string currentName = theStack.Name;
+ // If this is a CDATA element and the tag doesn't match,
+ // or isn't properly formed (junk after the name),
+ // restart CDATA mode and process the tag as characters.
+ if (cDataElements && (theStack.Flags & Schema.F_CDATA) != 0)
+ {
+ bool realTag = (length == currentName.Length);
+ if (realTag)
+ {
+ for (int i = 0; i < length; i++)
+ {
+ if (char.ToLower(buff[offset + i]) != char.ToLower(currentName[i]))
+ {
+ realTag = false;
+ break;
+ }
+ }
+ }
+ if (!realTag)
+ {
+ theContentHandler.Characters(etagchars, 0, 2);
+ theContentHandler.Characters(buff, offset, length);
+ theContentHandler.Characters(etagchars, 2, 1);
+ theScanner.StartCDATA();
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public virtual void ETagBasic(char[] buff, int offset, int length)
+ {
+ theNewElement = null;
+ string name;
+ if (length != 0)
+ {
+ // Canonicalize case of name
+ name = MakeName(buff, offset, length);
+ // System.err.println("got etag [" + name + "]");
+ ElementType type = theSchema.GetElementType(name);
+ if (type == null)
+ {
+ return; // mysterious end-tag
+ }
+ name = type.Name;
+ }
+ else
+ {
+ name = theStack.Name;
+ }
+ // System.err.println("%% Got end of " + name);
+
+ Element sp;
+ bool inNoforce = false;
+ for (sp = theStack; sp != null; sp = sp.Next)
+ {
+ if (sp.Name.Equals(name))
+ {
+ break;
+ }
+ if ((sp.Flags & Schema.F_NOFORCE) != 0)
+ {
+ inNoforce = true;
+ }
+ }
+
+ if (sp == null)
+ {
+ return; // Ignore unknown etags
+ }
+ if (sp.Next == null || sp.Next.Next == null)
+ {
+ return;
+ }
+ if (inNoforce)
+ {
+ // inside an F_NOFORCE element?
+ sp.Preclose(); // preclose the matching element
+ }
+ else
+ {
+ // restartably pop everything above us
+ while (theStack != sp)
+ {
+ RestartablyPop();
+ }
+ Pop();
+ }
+ // pop any preclosed elements now at the top
+ while (theStack.IsPreclosed)
+ {
+ Pop();
+ }
+ Restart(null);
+ }
+
+ /// <summary>
+ /// Push restartables on the stack if possible
+ /// e is the next element to be started, if we know what it is
+ /// </summary>
+ /// <param name="e"></param>
+ private void Restart(Element e)
+ {
+ while (theSaved != null && theStack.CanContain(theSaved) && (e == null || theSaved.CanContain(e)))
+ {
+ Element next = theSaved.Next;
+ Push(theSaved);
+ theSaved = next;
+ }
+ }
+
+ /// <summary>
+ /// Pop the stack irrevocably
+ /// </summary>
+ private void Pop()
+ {
+ if (theStack == null)
+ {
+ return; // empty stack
+ }
+ string name = theStack.Name;
+ string localName = theStack.LocalName;
+ string ns = theStack.Namespace;
+ string prefix = PrefixOf(name);
+
+ // System.err.println("%% Popping " + name);
+ if (!namespaces)
+ {
+ ns = localName = "";
+ }
+ theContentHandler.EndElement(ns, localName, name);
+ if (Foreign(prefix, ns))
+ {
+ theContentHandler.EndPrefixMapping(prefix);
+ // System.err.println("%% Unmapping [" + prefix + "] for elements to " + namespace);
+ }
+ Attributes atts = theStack.Attributes;
+ for (int i = atts.Length - 1; i >= 0; i--)
+ {
+ string attNamespace = atts.GetURI(i);
+ string attPrefix = PrefixOf(atts.GetQName(i));
+ if (Foreign(attPrefix, attNamespace))
+ {
+ theContentHandler.EndPrefixMapping(attPrefix);
+ // System.err.println("%% Unmapping [" + attPrefix + "] for attributes to " + attNamespace);
+ }
+ }
+ theStack = theStack.Next;
+ }
+
+ /// <summary>
+ /// Pop the stack restartably
+ /// </summary>
+ private void RestartablyPop()
+ {
+ Element popped = theStack;
+ Pop();
+ if (restartElements && (popped.Flags & Schema.F_RESTART) != 0)
+ {
+ popped.Anonymize();
+ popped.Next = theSaved;
+ theSaved = popped;
+ }
+ }
+
+ // Push element onto stack
+ private bool virginStack = true;
+ private void Push(Element e)
+ {
+ string name = e.Name;
+ string localName = e.LocalName;
+ string ns = e.Namespace;
+ string prefix = PrefixOf(name);
+
+ // System.err.println("%% Pushing " + name);
+ e.Clean();
+ if (!namespaces)
+ {
+ ns = localName = "";
+ }
+ if (virginStack && localName.Equals(theDoctypeName, StringComparison.OrdinalIgnoreCase))
+ {
+ try
+ {
+ theEntityResolver.ResolveEntity(theDoctypePublicId, theDoctypeSystemId);
+ }
+ catch (IOException)
+ {
+ } // Can't be thrown for root I believe.
+ }
+ if (Foreign(prefix, ns))
+ {
+ theContentHandler.StartPrefixMapping(prefix, ns);
+ // System.err.println("%% Mapping [" + prefix + "] for elements to " + namespace);
+ }
+ Attributes atts = e.Attributes;
+ int len = atts.Length;
+ for (int i = 0; i < len; i++)
+ {
+ string attNamespace = atts.GetURI(i);
+ string attPrefix = PrefixOf(atts.GetQName(i));
+ if (Foreign(attPrefix, attNamespace))
+ {
+ theContentHandler.StartPrefixMapping(attPrefix, attNamespace);
+ // System.err.println("%% Mapping [" + attPrefix + "] for attributes to " + attNamespace);
+ }
+ }
+ theContentHandler.StartElement(ns, localName, name, e.Attributes);
+ e.Next = theStack;
+ theStack = e;
+ virginStack = false;
+ if (cDataElements && (theStack.Flags & Schema.F_CDATA) != 0)
+ {
+ theScanner.StartCDATA();
+ }
+ }
+
+ /// <summary>
+ /// Get the prefix from a QName
+ /// </summary>
+ /// <param name="name"></param>
+ /// <returns></returns>
+ private static string PrefixOf(string name)
+ {
+ int i = name.IndexOf(':');
+ string prefix = "";
+ if (i != -1)
+ {
+ prefix = name.Substring(0, i);
+ }
+ // System.err.println("%% " + prefix + " is prefix of " + name);
+ return prefix;
+ }
+
+ /// <summary>
+ /// Return true if we have a foreign name
+ /// </summary>
+ /// <param name="prefix"></param>
+ /// <param name="ns"></param>
+ /// <returns></returns>
+ private bool Foreign(string prefix, string ns)
+ {
+ // System.err.print("%% Testing " + prefix + " and " + namespace + " for foreignness -- ");
+ bool foreign = !(prefix.Equals("") || ns.Equals("") || ns.Equals(theSchema.Uri));
+ // System.err.println(foreign);
+ return foreign;
+ }
+
+ /// <summary>
+ /// Parsing the complete XML Document Type Definition is way too complex,
+ /// but for many simple cases we can extract something useful from it.
+ /// doctypedecl ::= '<!DOCTYPE' S Name (S ExternalID)? S? ('[' intSubset ']' S?)? '>'
+ /// DeclSep ::= PEReference | S
+ /// intSubset ::= (markupdecl | DeclSep)*
+ /// markupdecl ::= elementdecl | AttlistDecl | EntityDecl | NotationDecl | PI | Comment
+ /// ExternalID ::= 'SYSTEM' S SystemLiteral | 'PUBLIC' S PubidLiteral S SystemLiteral
+ /// </summary>
+ /// <param name="buff"></param>
+ /// <param name="offset"></param>
+ /// <param name="length"></param>
+ public virtual void Decl(char[] buff, int offset, int length)
+ {
+ var s = new string(buff, offset, length);
+ string name = null;
+ string systemid = null;
+ string publicid = null;
+ string[] v = Split(s);
+ if (v.Length > 0 && "DOCTYPE".Equals(v[0], StringComparison.OrdinalIgnoreCase))
+ {
+ if (theDoctypeIsPresent)
+ {
+ return; // one doctype only!
+ }
+ theDoctypeIsPresent = true;
+ if (v.Length > 1)
+ {
+ name = v[1];
+ if (v.Length > 3 && "SYSTEM".Equals(v[2]))
+ {
+ systemid = v[3];
+ }
+ else if (v.Length > 3 && "PUBLIC".Equals(v[2]))
+ {
+ publicid = v[3];
+ if (v.Length > 4)
+ {
+ systemid = v[4];
+ }
+ else
+ {
+ systemid = "";
+ }
+ }
+ }
+ }
+ publicid = TrimQuotes(publicid);
+ systemid = TrimQuotes(systemid);
+ if (name != null)
+ {
+ publicid = CleanPublicId(publicid);
+ theLexicalHandler.StartDTD(name, publicid, systemid);
+ theLexicalHandler.EndDTD();
+ theDoctypeName = name;
+ theDoctypePublicId = publicid;
+ var locator = theScanner as ILocator;
+ if (locator != null)
+ {
+ // Must resolve systemid
+ theDoctypeSystemId = locator.SystemId;
+ try
+ {
+ if (Uri.IsWellFormedUriString(theDoctypeSystemId, UriKind.Absolute))
+ {
+ theDoctypeSystemId = new Uri(new Uri(theDoctypeSystemId), systemid).ToString();
+ }
+ }
+ catch (Exception)
+ {
+ }
+ }
+ }
+ }
+
+ // If the string is quoted, trim the quotes.
+ private static string TrimQuotes(string value)
+ {
+ if (value == null)
+ {
+ return null;
+ }
+ int length = value.Length;
+ if (length == 0)
+ {
+ return value;
+ }
+ char s = value[0];
+ char e = value[length - 1];
+ if (s == e && (s == '\'' || s == '"'))
+ {
+ value = value.Substring(1, value.Length - 1);
+ }
+ return value;
+ }
+
+ /// <summary>
+ /// Split the supplied string into words or phrases seperated by spaces.
+ /// Recognises quotes around a phrase and doesn't split it.
+ /// </summary>
+ /// <param name="val"></param>
+ /// <returns></returns>
+ private static string[] Split(string val)
+ {
+ val = val.Trim();
+ if (val.Length == 0)
+ {
+ return new string[0];
+ }
+ var l = new List<string>();
+ int s = 0;
+ int e = 0;
+ bool sq = false; // single quote
+ bool dq = false; // double quote
+ var lastc = (char)0;
+ int len = val.Length;
+ for (e = 0; e < len; e++)
+ {
+ char c = val[e];
+ if (!dq && c == '\'' && lastc != '\\')
+ {
+ sq = !sq;
+ if (s < 0)
+ {
+ s = e;
+ }
+ }
+ else if (!sq && c == '\"' && lastc != '\\')
+ {
+ dq = !dq;
+ if (s < 0)
+ {
+ s = e;
+ }
+ }
+ else if (!sq && !dq)
+ {
+ if (char.IsWhiteSpace(c))
+ {
+ if (s >= 0)
+ {
+ l.Add(val.Substring(s, e - s));
+ }
+ s = -1;
+ }
+ else if (s < 0 && c != ' ')
+ {
+ s = e;
+ }
+ }
+ lastc = c;
+ }
+ l.Add(val.Substring(s, e - s));
+ return l.ToArray();
+ }
+
+ private const string LEGAL = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-'()+,./:=?;!*#@$_%";
+
+ /// <summary>
+ /// Replace junk in publicids with spaces
+ /// </summary>
+ /// <param name="src"></param>
+ /// <returns></returns>
+ private string CleanPublicId(string src)
+ {
+ if (src == null)
+ {
+ return null;
+ }
+ int len = src.Length;
+ var dst = new StringBuilder(len);
+ bool suppressSpace = true;
+ for (int i = 0; i < len; i++)
+ {
+ char ch = src[i];
+ if (LEGAL.IndexOf(ch) != -1)
+ {
+ // legal but not whitespace
+ dst.Append(ch);
+ suppressSpace = false;
+ }
+ else if (suppressSpace)
+ {
+ // normalizable whitespace or junk
+ }
+ else
+ {
+ dst.Append(' ');
+ suppressSpace = true;
+ }
+ }
+ // System.err.println("%% Publicid [" + dst.tostring().trim() + "]");
+ return dst.ToString().Trim(); // trim any final junk whitespace
+ }
+
+ public virtual void GI(char[] buff, int offset, int length)
+ {
+ if (theNewElement != null)
+ {
+ return;
+ }
+ string name = MakeName(buff, offset, length);
+ if (name == null)
+ {
+ return;
+ }
+ ElementType type = theSchema.GetElementType(name);
+ if (type == null)
+ {
+ // Suppress unknown elements if ignore-bogons is on
+ if (ignoreBogons)
+ {
+ return;
+ }
+ int bogonModel = (bogonsEmpty ? Schema.M_EMPTY : Schema.M_ANY);
+ int bogonMemberOf = (rootBogons ? Schema.M_ANY : (Schema.M_ANY & ~Schema.M_ROOT));
+ theSchema.ElementType(name, bogonModel, bogonMemberOf, 0);
+ if (!rootBogons)
+ {
+ theSchema.Parent(name, theSchema.RootElementType.Name);
+ }
+ type = theSchema.GetElementType(name);
+ }
+
+ theNewElement = new Element(type, defaultAttributes);
+ // System.err.println("%% Got GI " + theNewElement.name());
+ }
+
+ public virtual void CDSect(char[] buff, int offset, int length)
+ {
+ theLexicalHandler.StartCDATA();
+ PCDATA(buff, offset, length);
+ theLexicalHandler.EndCDATA();
+ }
+
+ public virtual void PCDATA(char[] buff, int offset, int length)
+ {
+ if (length == 0)
+ {
+ return;
+ }
+ bool allWhite = true;
+ for (int i = 0; i < length; i++)
+ {
+ if (!char.IsWhiteSpace(buff[offset + i]))
+ {
+ allWhite = false;
+ }
+ }
+ if (allWhite && !theStack.CanContain(thePCDATA))
+ {
+ if (ignorableWhitespace)
+ {
+ theContentHandler.IgnorableWhitespace(buff, offset, length);
+ }
+ }
+ else
+ {
+ Rectify(thePCDATA);
+ theContentHandler.Characters(buff, offset, length);
+ }
+ }
+
+ public virtual void PITarget(char[] buff, int offset, int length)
+ {
+ if (theNewElement != null)
+ {
+ return;
+ }
+ thePITarget = MakeName(buff, offset, length).Replace(':', '_');
+ }
+
+ public virtual void PI(char[] buff, int offset, int length)
+ {
+ if (theNewElement != null || thePITarget == null)
+ {
+ return;
+ }
+ if ("xml".Equals(thePITarget, StringComparison.OrdinalIgnoreCase))
+ {
+ return;
+ }
+ // if (length > 0 && buff[length - 1] == '?') System.err.println("%% Removing ? from PI");
+ if (length > 0 && buff[length - 1] == '?')
+ {
+ length--; // remove trailing ?
+ }
+ theContentHandler.ProcessingInstruction(thePITarget, new string(buff, offset, length));
+ thePITarget = null;
+ }
+
+ public virtual void STagC(char[] buff, int offset, int length)
+ {
+ // System.err.println("%% Start-tag");
+ if (theNewElement == null)
+ {
+ return;
+ }
+ Rectify(theNewElement);
+ if (theStack.Model == Schema.M_EMPTY)
+ {
+ // Force an immediate end tag
+ ETagBasic(buff, offset, length);
+ }
+ }
+
+ public virtual void STagE(char[] buff, int offset, int length)
+ {
+ // System.err.println("%% Empty-tag");
+ if (theNewElement == null)
+ {
+ return;
+ }
+ Rectify(theNewElement);
+ // Force an immediate end tag
+ ETagBasic(buff, offset, length);
+ }
+
+ private char[] theCommentBuffer = new char[2000];
+ public virtual void Cmnt(char[] buff, int offset, int length)
+ {
+ theLexicalHandler.Comment(buff, offset, length);
+ }
+
+ /// <summary>
+ /// Rectify the stack, pushing and popping as needed
+ /// so that the argument can be safely pushed
+ /// </summary>
+ /// <param name="e"></param>
+ private void Rectify(Element e)
+ {
+ Element sp;
+ while (true)
+ {
+ for (sp = theStack; sp != null; sp = sp.Next)
+ {
+ if (sp.CanContain(e))
+ {
+ break;
+ }
+ }
+ if (sp != null)
+ {
+ break;
+ }
+ ElementType parentType = e.Parent;
+ if (parentType == null)
+ {
+ break;
+ }
+ var parent = new Element(parentType, defaultAttributes);
+ // System.err.println("%% Ascending from " + e.name() + " to " + parent.name());
+ parent.Next = e;
+ e = parent;
+ }
+ if (sp == null)
+ {
+ return; // don't know what to do
+ }
+ while (theStack != sp)
+ {
+ if (theStack == null || theStack.Next == null || theStack.Next.Next == null)
+ {
+ break;
+ }
+ RestartablyPop();
+ }
+ while (e != null)
+ {
+ Element nexte = e.Next;
+ if (!e.Name.Equals("<pcdata>"))
+ {
+ Push(e);
+ }
+ e = nexte;
+ Restart(e);
+ }
+ theNewElement = null;
+ }
+
+ public virtual int GetEntity()
+ {
+ return theEntity;
+ }
+
+ /// <summary>
+ /// Return the argument as a valid XML name
+ /// This no longer lowercases the result: we depend on Schema to
+ /// canonicalize case.
+ /// </summary>
+ /// <param name="buff"></param>
+ /// <param name="offset"></param>
+ /// <param name="length"></param>
+ /// <returns></returns>
+ private string MakeName(char[] buff, int offset, int length)
+ {
+ var dst = new StringBuilder(length + 2);
+ bool seenColon = false;
+ bool start = true;
+ // string src = new string(buff, offset, length); // DEBUG
+ for (; length-- > 0; offset++)
+ {
+ char ch = buff[offset];
+ if (char.IsLetter(ch) || ch == '_')
+ {
+ start = false;
+ dst.Append(ch);
+ }
+ else if (char.IsDigit(ch) || ch == '-' || ch == '.')
+ {
+ if (start)
+ {
+ dst.Append('_');
+ }
+ start = false;
+ dst.Append(ch);
+ }
+ else if (ch == ':' && !seenColon)
+ {
+ seenColon = true;
+ if (start)
+ {
+ dst.Append('_');
+ }
+ start = true;
+ dst.Append(translateColons ? '_' : ch);
+ }
+ }
+ int dstLength = dst.Length;
+ if (dstLength == 0 || dst[dstLength - 1] == ':')
+ {
+ dst.Append('_');
+ }
+ // System.err.println("Made name \"" + dst + "\" from \"" + src + "\"");
+ return dst.ToString().Intern();
+ }
+
+ private class AutoDetectorDelegate : IAutoDetector
+ {
+ private readonly Func<Stream, StreamReader> _delegate;
+
+ public AutoDetectorDelegate(Func<Stream, StreamReader> @delegate)
+ {
+ _delegate = @delegate;
+ }
+
+ public TextReader AutoDetectingReader(Stream stream)
+ {
+ return _delegate(stream);
+ }
+ }
+
+ // Default LexicalHandler implementation
+
+ public virtual void Comment(char[] ch, int start, int length)
+ {
+ }
+
+ public virtual void EndCDATA()
+ {
+ }
+
+ public virtual void EndDTD()
+ {
+ }
+
+ public virtual void EndEntity(string name)
+ {
+ }
+
+ public virtual void StartCDATA()
+ {
+ }
+
+ public virtual void StartDTD(string name, string publicid, string systemid)
+ {
+ }
+
+ public virtual void StartEntity(string name)
+ {
+ }
+
+ /// <summary>
+ /// Creates a new instance of <see cref="Parser" />
+ /// </summary>
+ public Parser()
+ {
+ theNewElement = null;
+ theContentHandler = this;
+ theLexicalHandler = this;
+ theDTDHandler = this;
+ theErrorHandler = this;
+ theEntityResolver = this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/ScanHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/ScanHandler.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/ScanHandler.cs
new file mode 100644
index 0000000..3901ada
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/ScanHandler.cs
@@ -0,0 +1,105 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// Scanner handler
+
+namespace TagSoup
+{
+ /// <summary>
+ /// An interface that Scanners use to report events in the input stream.
+ /// </summary>
+ public interface IScanHandler
+ {
+ /// <summary>
+ /// Reports an attribute name without a value.
+ /// </summary>
+ void Adup(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports an attribute name; a value will follow.
+ /// </summary>
+ void Aname(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports an attribute value.
+ /// </summary>
+ void Aval(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports the content of a CDATA section (not a CDATA element)
+ /// </summary>
+ void CDSect(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports a <!....> declaration - typically a DOCTYPE
+ /// </summary>
+ void Decl(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports an entity reference or character reference.
+ /// </summary>
+ void Entity(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports EOF.
+ /// </summary>
+ void EOF(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports an end-tag.
+ /// </summary>
+ void ETag(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports the general identifier (element type name) of a start-tag.
+ /// </summary>
+ void GI(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports character content.
+ /// </summary>
+ void PCDATA(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports the data part of a processing instruction.
+ /// </summary>
+ void PI(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports the target part of a processing instruction.
+ /// </summary>
+ void PITarget(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports the close of a start-tag.
+ /// </summary>
+ void STagC(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports the close of an empty-tag.
+ /// </summary>
+ void STagE(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Reports a comment.
+ /// </summary>
+ void Cmnt(char[] buff, int offset, int length);
+
+ /// <summary>
+ /// Returns the value of the last entity or character reference reported.
+ /// </summary>
+ /// <returns>The value of the last entity or character reference reported.</returns>
+ int GetEntity();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/Scanner.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/Scanner.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/Scanner.cs
new file mode 100644
index 0000000..5e4d406
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/Scanner.cs
@@ -0,0 +1,53 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// Scanner
+
+using System.IO;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// An interface allowing <see cref="Parser"/> to invoke scanners.
+ /// </summary>
+ public interface IScanner
+ {
+ /// <summary>
+ /// Invoke a scanner.
+ /// </summary>
+ /// <param name="br">
+ /// A source of characters to scan
+ /// </param>
+ /// <param name="handler">
+ /// A <see cref="IScanHandler"/> to report events to
+ /// </param>
+ void Scan(TextReader br, IScanHandler handler);
+
+ /// <summary>
+ /// Reset the embedded locator.
+ /// </summary>
+ /// <param name="publicid">
+ /// The publicid of the source
+ /// </param>
+ /// <param name="systemid">
+ /// The systemid of the source
+ /// </param>
+ void ResetDocumentLocator(string publicid, string systemid);
+
+ /// <summary>
+ /// Signal to the scanner to start CDATA content mode.
+ /// </summary>
+ void StartCDATA();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/Schema.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/Schema.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/Schema.cs
new file mode 100644
index 0000000..76a86f9
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/Schema.cs
@@ -0,0 +1,159 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// Model of document
+
+using System;
+using System.Collections;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// Abstract class representing a TSSL schema.
+ /// Actual TSSL schemas are compiled into concrete subclasses of this class.
+ /// </summary>
+ public abstract class Schema
+ {
+ public const int M_ANY = -1;//0xFFFFFFFF;
+ public const int M_EMPTY = 0;
+ public const int M_PCDATA = 1 << 30;
+ public const int M_ROOT = 1 << 31;
+
+ public const int F_RESTART = 1;
+ public const int F_CDATA = 2;
+ public const int F_NOFORCE = 4;
+
+ private readonly Hashtable theEntities = new Hashtable(); // string -> Character
+ private readonly Hashtable theElementTypes = new Hashtable(); // string -> ElementType
+
+ private string theURI = "";
+ private string thePrefix = "";
+ private ElementType theRoot;
+
+ /// <summary>
+ /// Add or replace an element type for this schema.
+ /// </summary>
+ /// <param name="name"> Name (Qname) of the element</param>
+ /// <param name="model">Models of the element's content as a vector of bits</param>
+ /// <param name="memberOf">Models the element is a member of as a vector of bits</param>
+ /// <param name="flags">Flags for the element</param>
+ public virtual void ElementType(string name, int model, int memberOf, int flags)
+ {
+ var e = new ElementType(name, model, memberOf, flags, this);
+ theElementTypes[name.ToLower()] = e;
+ if (memberOf == M_ROOT)
+ {
+ theRoot = e;
+ }
+ }
+
+ /// <summary>
+ /// Gets or sets the root element of this schema
+ /// </summary>
+ public virtual ElementType RootElementType
+ {
+ get { return theRoot; }
+ }
+
+ /// <summary>
+ /// Add or replace a default attribute for an element type in this schema.
+ /// </summary>
+ /// <param name="elemName">Name (Qname) of the element type</param>
+ /// <param name="attrName">Name (Qname) of the attribute</param>
+ /// <param name="type">Type of the attribute</param>
+ /// <param name="value">Default value of the attribute; null if no default</param>
+ public virtual void Attribute(string elemName, string attrName, string type, string value)
+ {
+ ElementType e = GetElementType(elemName);
+ if (e == null)
+ {
+ throw new Exception("Attribute " + attrName + " specified for unknown element type " + elemName);
+ }
+ e.SetAttribute(attrName, type, value);
+ }
+
+ /// <summary>
+ /// Specify natural parent of an element in this schema.
+ /// </summary>
+ /// <param name="name">Name of the child element</param>
+ /// <param name="parentName">Name of the parent element</param>
+ public virtual void Parent(string name, string parentName)
+ {
+ ElementType child = GetElementType(name);
+ ElementType parent = GetElementType(parentName);
+ if (child == null)
+ {
+ throw new Exception("No child " + name + " for parent " + parentName);
+ }
+ if (parent == null)
+ {
+ throw new Exception("No parent " + parentName + " for child " + name);
+ }
+ child.Parent = parent;
+ }
+
+ /// <summary>
+ /// Add to or replace a character entity in this schema.
+ /// </summary>
+ /// <param name="name">Name of the entity</param>
+ /// <param name="value">Value of the entity</param>
+ public virtual void Entity(string name, int value)
+ {
+ theEntities[name] = value;
+ }
+
+ /// <summary>
+ /// Get an <see cref="TagSoup.ElementType"/> by name.
+ /// </summary>
+ /// <param name="name">Name (Qname) of the element type</param>
+ /// <returns>The corresponding <see cref="TagSoup.ElementType"/></returns>
+ public virtual ElementType GetElementType(string name)
+ {
+ return (ElementType)(theElementTypes[name.ToLower()]);
+ }
+
+ /// <summary>
+ /// Get an entity value by name.
+ /// </summary>
+ /// <param name="name">Name of the entity</param>
+ /// <returns>The corresponding character, or 0 if none</returns>
+ public virtual int GetEntity(string name)
+ {
+ // System.err.println("%% Looking up entity " + name);
+ if (theEntities.ContainsKey(name))
+ {
+ return (int)theEntities[name];
+ }
+ return 0;
+ }
+
+ /// <summary>
+ /// Gets or sets the URI (namespace name) of this schema.
+ /// </summary>
+ public virtual string Uri
+ {
+ get { return theURI; }
+ set { theURI = value; }
+ }
+
+ /// <summary>
+ /// Gets ot sets the prefix of this schema.
+ /// </summary>
+ public virtual string Prefix
+ {
+ get { return thePrefix; }
+ set { thePrefix = value; }
+ }
+ }
+}
[10/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/SortableSingleDocSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/SortableSingleDocSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/SortableSingleDocSource.cs
new file mode 100644
index 0000000..c83828c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/SortableSingleDocSource.cs
@@ -0,0 +1,114 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Adds fields appropriate for sorting: country, random_string and sort_field
+ /// (int). Supports the following parameters:
+ /// <list type="bullet">
+ /// <item><term><b>sort.rng</b></term><description>defines the range for sort-by-int field (default <b>20000</b>).</description></item>
+ /// <item><term><b>rand.seed</b></term><description>defines the seed to initialize Random with (default <b>13</b>).</description></item>
+ /// </list>
+ /// </summary>
+ public class SortableSingleDocSource : SingleDocSource
+ {
+ private static string[] COUNTRIES = new string[] {
+ "European Union", "United States", "Japan", "Germany", "China (PRC)",
+ "United Kingdom", "France", "Italy", "Spain", "Canada", "Brazil", "Russia",
+ "India", "South Korea", "Australia", "Mexico", "Netherlands", "Turkey",
+ "Sweden", "Belgium", "Indonesia", "Switzerland", "Poland", "Norway",
+ "Republic of China", "Saudi Arabia", "Austria", "Greece", "Denmark", "Iran",
+ "South Africa", "Argentina", "Ireland", "Thailand", "Finland", "Venezuela",
+ "Portugal", "Hong Kong", "United Arab Emirates", "Malaysia",
+ "Czech Republic", "Colombia", "Nigeria", "Romania", "Chile", "Israel",
+ "Singapore", "Philippines", "Pakistan", "Ukraine", "Hungary", "Algeria",
+ "New Zealand", "Egypt", "Kuwait", "Peru", "Kazakhstan", "Slovakia",
+ "Morocco", "Bangladesh", "Vietnam", "Qatar", "Angola", "Libya", "Iraq",
+ "Croatia", "Luxembourg", "Sudan", "Slovenia", "Cuba", "Belarus", "Ecuador",
+ "Serbia", "Oman", "Bulgaria", "Lithuania", "Syria", "Dominican Republic",
+ "Tunisia", "Guatemala", "Azerbaijan", "Sri Lanka", "Kenya", "Latvia",
+ "Turkmenistan", "Costa Rica", "Lebanon", "Uruguay", "Uzbekistan", "Yemen",
+ "Cyprus", "Estonia", "Trinidad and Tobago", "Cameroon", "El Salvador",
+ "Iceland", "Panama", "Bahrain", "Ivory Coast", "Ethiopia", "Tanzania",
+ "Jordan", "Ghana", "Bosnia and Herzegovina", "Macau", "Burma", "Bolivia",
+ "Brunei", "Botswana", "Honduras", "Gabon", "Uganda", "Jamaica", "Zambia",
+ "Senegal", "Paraguay", "Albania", "Equatorial Guinea", "Georgia",
+ "Democratic Republic of the Congo", "Nepal", "Afghanistan", "Cambodia",
+ "Armenia", "Republic of the Congo", "Mozambique", "Republic of Macedonia",
+ "Malta", "Namibia", "Madagascar", "Chad", "Burkina Faso", "Mauritius",
+ "Mali", "The Bahamas", "Papua New Guinea", "Nicaragua", "Haiti", "Benin",
+ "alestinian flag West Bank and Gaza", "Jersey", "Fiji", "Guinea", "Moldova",
+ "Niger", "Laos", "Mongolia", "French Polynesia", "Kyrgyzstan", "Barbados",
+ "Tajikistan", "Malawi", "Liechtenstein", "New Caledonia", "Kosovo",
+ "Rwanda", "Montenegro", "Swaziland", "Guam", "Mauritania", "Guernsey",
+ "Isle of Man", "Togo", "Somalia", "Suriname", "Aruba", "North Korea",
+ "Zimbabwe", "Central African Republic", "Faroe Islands", "Greenland",
+ "Sierra Leone", "Lesotho", "Cape Verde", "Eritrea", "Bhutan", "Belize",
+ "Antigua and Barbuda", "Gibraltar", "Maldives", "San Marino", "Guyana",
+ "Burundi", "Saint Lucia", "Djibouti", "British Virgin Islands", "Liberia",
+ "Seychelles", "The Gambia", "Northern Mariana Islands", "Grenada",
+ "Saint Vincent and the Grenadines", "Saint Kitts and Nevis", "East Timor",
+ "Vanuatu", "Comoros", "Samoa", "Solomon Islands", "Guinea-Bissau",
+ "American Samoa", "Dominica", "Micronesia", "Tonga", "Cook Islands",
+ "Palau", "Marshall Islands", "S�o Tom� and Pr�ncipe", "Anguilla",
+ "Kiribati", "Tuvalu", "Niue" };
+
+ private int sortRange;
+ private Random r;
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ docData = base.GetNextDocData(docData);
+ var props = new Dictionary<string, string>();
+
+ // random int
+ props["sort_field"] = r.Next(sortRange).ToString(CultureInfo.InvariantCulture);
+
+ // random string
+ int len = NextInt32(2, 20);
+ char[] buffer = new char[len];
+ for (int i = 0; i < len; i++)
+ {
+ buffer[i] = (char)r.Next(0x80);
+ }
+ props["random_string"] = new string(buffer);
+
+ // random country
+ props["country"] = COUNTRIES[r.Next(COUNTRIES.Length)];
+ docData.Props = props;
+ return docData;
+ }
+
+ private int NextInt32(int start, int end)
+ {
+ return start + r.Next(end - start);
+ }
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+ sortRange = config.Get("sort.rng", 20000);
+ r = new Random(config.Get("rand.seed", 13));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialDocMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialDocMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialDocMaker.cs
new file mode 100644
index 0000000..7879cd8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialDocMaker.cs
@@ -0,0 +1,249 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Spatial;
+using Lucene.Net.Spatial.Prefix;
+using Lucene.Net.Spatial.Prefix.Tree;
+using Lucene.Net.Support;
+using Spatial4n.Core.Context;
+using Spatial4n.Core.Shapes;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Indexes spatial data according to a configured <see cref="SpatialStrategy"/> with optional
+ /// shape transformation via a configured <see cref="IShapeConverter"/>. The converter can turn points into
+ /// circles and bounding boxes, in order to vary the type of indexing performance tests.
+ /// Unless it's subclass-ed to do otherwise, this class configures a <see cref="SpatialContext"/>,
+ /// <see cref="SpatialPrefixTree"/>, and <see cref="RecursivePrefixTreeStrategy"/>. The Strategy is made
+ /// available to a query maker via the static method <see cref="GetSpatialStrategy(int)"/>.
+ /// See spatial.alg for a listing of spatial parameters, in particular those starting with "spatial."
+ /// and "doc.spatial".
+ /// </summary>
+ public class SpatialDocMaker : DocMaker
+ {
+ public static readonly string SPATIAL_FIELD = "spatial";
+
+ //cache spatialStrategy by round number
+ private static IDictionary<int, SpatialStrategy> spatialStrategyCache = new Dictionary<int, SpatialStrategy>();
+
+ private SpatialStrategy strategy;
+ private IShapeConverter shapeConverter;
+
+ /// <summary>
+ /// Looks up the <see cref="SpatialStrategy"/> from the given round --
+ /// <see cref="Config.RoundNumber"/>. It's an error
+ /// if it wasn't created already for this round -- when <see cref="SpatialDocMaker"/> is initialized.
+ /// </summary>
+ public static SpatialStrategy GetSpatialStrategy(int roundNumber)
+ {
+ SpatialStrategy result;
+ if (!spatialStrategyCache.TryGetValue(roundNumber, out result) || result == null)
+ {
+ throw new InvalidOperationException("Strategy should have been init'ed by SpatialDocMaker by now");
+ }
+ return result;
+ }
+
+ /// <summary>
+ /// Builds a <see cref="SpatialStrategy"/> from configuration options.
+ /// </summary>
+ protected virtual SpatialStrategy MakeSpatialStrategy(Config config)
+ {
+ //A Map view of Config that prefixes keys with "spatial."
+ var configMap = new DictionaryAnonymousHelper(config);
+
+ SpatialContext ctx = SpatialContextFactory.MakeSpatialContext(configMap /*, null*/); // LUCENENET TODO: What is this extra param?
+
+ //Some day the strategy might be initialized with a factory but such a factory
+ // is non-existent.
+ return MakeSpatialStrategy(config, configMap, ctx);
+ }
+
+ private class DictionaryAnonymousHelper : Dictionary<string, string>
+ {
+ private readonly Config config;
+ public DictionaryAnonymousHelper(Config config)
+ {
+ this.config = config;
+ }
+
+ // LUCENENET TODO: EntrySet not supported. Should we throw on GetEnumerator()?
+
+ new public string this[string key]
+ {
+ get { return config.Get("spatial." + key, null); }
+ }
+ }
+
+ protected virtual SpatialStrategy MakeSpatialStrategy(Config config, IDictionary<string, string> configMap,
+ SpatialContext ctx)
+ {
+ //A factory for the prefix tree grid
+ SpatialPrefixTree grid = SpatialPrefixTreeFactory.MakeSPT(configMap, /*null,*/ ctx); // LUCENENET TODO: What is this extra param?
+
+ RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategyAnonymousHelper(grid, SPATIAL_FIELD, config);
+
+ int prefixGridScanLevel = config.Get("query.spatial.prefixGridScanLevel", -4);
+ if (prefixGridScanLevel < 0)
+ prefixGridScanLevel = grid.MaxLevels + prefixGridScanLevel;
+ strategy.PrefixGridScanLevel = prefixGridScanLevel;
+
+ double distErrPct = config.Get("spatial.distErrPct", .025);//doc & query; a default
+ strategy.DistErrPct = distErrPct;
+ return strategy;
+ }
+
+ private class RecursivePrefixTreeStrategyAnonymousHelper : RecursivePrefixTreeStrategy
+ {
+ public RecursivePrefixTreeStrategyAnonymousHelper(SpatialPrefixTree grid, string fieldName, Config config)
+ : base(grid, fieldName)
+ {
+ this.m_pointsOnly = config.Get("spatial.docPointsOnly", false);
+ }
+ }
+
+ public override void SetConfig(Config config, ContentSource source)
+ {
+ base.SetConfig(config, source);
+ SpatialStrategy existing;
+ if (!spatialStrategyCache.TryGetValue(config.RoundNumber, out existing) || existing == null)
+ {
+ //new round; we need to re-initialize
+ strategy = MakeSpatialStrategy(config);
+ spatialStrategyCache[config.RoundNumber] = strategy;
+ //TODO remove previous round config?
+ shapeConverter = MakeShapeConverter(strategy, config, "doc.spatial.");
+ SystemConsole.WriteLine("Spatial Strategy: " + strategy);
+ }
+ }
+
+ /// <summary>
+ /// Optionally converts points to circles, and optionally bbox'es result.
+ /// </summary>
+ public static IShapeConverter MakeShapeConverter(SpatialStrategy spatialStrategy,
+ Config config, string configKeyPrefix)
+ {
+ //by default does no conversion
+ double radiusDegrees = config.Get(configKeyPrefix + "radiusDegrees", 0.0);
+ double plusMinus = config.Get(configKeyPrefix + "radiusDegreesRandPlusMinus", 0.0);
+ bool bbox = config.Get(configKeyPrefix + "bbox", false);
+
+ return new ShapeConverterAnonymousHelper(spatialStrategy, radiusDegrees, plusMinus, bbox);
+ }
+
+ private class ShapeConverterAnonymousHelper : IShapeConverter
+ {
+ private readonly SpatialStrategy spatialStrategy;
+ private readonly double radiusDegrees;
+ private readonly double plusMinus;
+ private readonly bool bbox;
+
+ public ShapeConverterAnonymousHelper(SpatialStrategy spatialStrategy, double radiusDegrees, double plusMinus, bool bbox)
+ {
+ this.spatialStrategy = spatialStrategy;
+ this.radiusDegrees = radiusDegrees;
+ this.plusMinus = plusMinus;
+ this.bbox = bbox;
+ }
+
+ public IShape Convert(IShape shape)
+ {
+ if (shape is IPoint && (radiusDegrees != 0.0 || plusMinus != 0.0))
+ {
+ IPoint point = (IPoint)shape;
+ double radius = radiusDegrees;
+ if (plusMinus > 0.0)
+ {
+ Random random = new Random(point.GetHashCode());//use hashCode so it's reproducibly random
+ radius += random.NextDouble() * 2 * plusMinus - plusMinus;
+ radius = Math.Abs(radius);//can happen if configured plusMinus > radiusDegrees
+ }
+ shape = spatialStrategy.SpatialContext.MakeCircle(point, radius);
+ }
+ if (bbox)
+ {
+ shape = shape.BoundingBox;
+ }
+ return shape;
+ }
+ }
+
+ // LUCENENET specific: de-nested IShapeConverter
+
+ public override Document MakeDocument()
+ {
+
+ DocState docState = GetDocState();
+
+ Document doc = base.MakeDocument();
+
+ // Set SPATIAL_FIELD from body
+ DocData docData = docState.docData;
+ // makeDocument() resets docState.getBody() so we can't look there; look in Document
+ string shapeStr = doc.GetField(DocMaker.BODY_FIELD).GetStringValue();
+ IShape shape = MakeShapeFromString(strategy, docData.Name, shapeStr);
+ if (shape != null)
+ {
+ shape = shapeConverter.Convert(shape);
+ //index
+ foreach (Field f in strategy.CreateIndexableFields(shape))
+ {
+ doc.Add(f);
+ }
+ }
+
+ return doc;
+ }
+
+ public static IShape MakeShapeFromString(SpatialStrategy strategy, string name, string shapeStr)
+ {
+ if (shapeStr != null && shapeStr.Length > 0)
+ {
+ try
+ {
+ return strategy.SpatialContext.ReadShapeFromWkt(shapeStr);
+ }
+ catch (Exception e)
+ {//InvalidShapeException TODO
+ SystemConsole.Error.WriteLine("Shape " + name + " wasn't parseable: " + e + " (skipping it)");
+ return null;
+ }
+ }
+ return null;
+ }
+
+ public override Document MakeDocument(int size)
+ {
+ //TODO consider abusing the 'size' notion to number of shapes per document
+ throw new NotSupportedException();
+ }
+ }
+
+ /// <summary>
+ /// Converts one shape to another. Created by
+ /// <see cref="MakeShapeConverter(SpatialStrategy, Config, string)"/>.
+ /// </summary>
+ public interface IShapeConverter
+ {
+ IShape Convert(IShape shape);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialFileQueryMaker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialFileQueryMaker.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialFileQueryMaker.cs
new file mode 100644
index 0000000..d583d22
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/SpatialFileQueryMaker.cs
@@ -0,0 +1,131 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Queries;
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Search;
+using Lucene.Net.Spatial;
+using Lucene.Net.Spatial.Queries;
+using Spatial4n.Core.Shapes;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Reads spatial data from the body field docs from an internally created <see cref="LineDocSource"/>.
+ /// It's parsed by <see cref="Spatial4n.Core.Context.SpatialContext.ReadShapeFromWkt(string)"/> and then
+ /// further manipulated via a configurable <see cref="IShapeConverter"/>. When using point
+ /// data, it's likely you'll want to configure the shape converter so that the query shapes actually
+ /// cover a region. The queries are all created & cached in advance. This query maker works in
+ /// conjunction with <see cref="SpatialDocMaker"/>. See spatial.alg for a listing of options, in
+ /// particular the options starting with "query.".
+ /// </summary>
+ public class SpatialFileQueryMaker : AbstractQueryMaker
+ {
+ protected SpatialStrategy m_strategy;
+ protected double m_distErrPct;//NaN if not set
+ protected SpatialOperation m_operation;
+ protected bool m_score;
+
+ protected IShapeConverter m_shapeConverter;
+
+ public override void SetConfig(Config config)
+ {
+ m_strategy = SpatialDocMaker.GetSpatialStrategy(config.RoundNumber);
+ m_shapeConverter = SpatialDocMaker.MakeShapeConverter(m_strategy, config, "query.spatial.");
+
+ m_distErrPct = config.Get("query.spatial.distErrPct", double.NaN);
+ m_operation = SpatialOperation.Get(config.Get("query.spatial.predicate", "Intersects"));
+ m_score = config.Get("query.spatial.score", false);
+
+ base.SetConfig(config);//call last, will call prepareQueries()
+ }
+
+ protected override Query[] PrepareQueries()
+ {
+ int maxQueries = m_config.Get("query.file.maxQueries", 1000);
+ Config srcConfig = new Config(new Dictionary<string, string>());
+ srcConfig.Set("docs.file", m_config.Get("query.file", null));
+ srcConfig.Set("line.parser", m_config.Get("query.file.line.parser", null));
+ srcConfig.Set("content.source.forever", "false");
+
+ List<Query> queries = new List<Query>();
+ LineDocSource src = new LineDocSource();
+ try
+ {
+ src.SetConfig(srcConfig);
+ src.ResetInputs();
+ DocData docData = new DocData();
+ for (int i = 0; i < maxQueries; i++)
+ {
+ docData = src.GetNextDocData(docData);
+ IShape shape = SpatialDocMaker.MakeShapeFromString(m_strategy, docData.Name, docData.Body);
+ if (shape != null)
+ {
+ shape = m_shapeConverter.Convert(shape);
+ queries.Add(MakeQueryFromShape(shape));
+ }
+ else
+ {
+ i--;//skip
+ }
+ }
+ }
+#pragma warning disable 168
+ catch (NoMoreDataException e)
+#pragma warning restore 168
+ {
+ //all-done
+ }
+ finally
+ {
+ src.Dispose();
+ }
+ return queries.ToArray();
+ }
+
+
+ protected virtual Query MakeQueryFromShape(IShape shape)
+ {
+ SpatialArgs args = new SpatialArgs(m_operation, shape);
+ if (!double.IsNaN(m_distErrPct))
+ args.DistErrPct = m_distErrPct;
+
+ if (m_score)
+ {
+ ValueSource valueSource = m_strategy.MakeDistanceValueSource(shape.Center);
+ return new CustomScoreQuery(m_strategy.MakeQuery(args), new FunctionQuery(valueSource));
+ }
+ else
+ {
+ //strategy.makeQuery() could potentially score (isn't well defined) so instead we call
+ // makeFilter() and wrap
+
+ Filter filter = m_strategy.MakeFilter(args);
+ if (filter is QueryWrapperFilter)
+ {
+ return ((QueryWrapperFilter)filter).Query;
+ }
+ else
+ {
+ return new ConstantScoreQuery(filter);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecContentSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecContentSource.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecContentSource.cs
new file mode 100644
index 0000000..d84a25d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecContentSource.cs
@@ -0,0 +1,350 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Implements a <see cref="ContentSource"/> over the TREC collection.
+ /// </summary>
+ /// <remarks>
+ /// Supports the following configuration parameters (on top of
+ /// <see cref="ContentSource"/>):
+ /// <list type="bullet">
+ /// <item><term>work.dir</term><description>specifies the working directory. Required if "docs.dir"
+ /// denotes a relative path (<b>default=work</b>).</description></item>
+ /// <item><term>docs.dir</term><description>specifies the directory where the TREC files reside.
+ /// Can be set to a relative path if "work.dir" is also specified
+ /// (<b>default=trec</b>).
+ /// </description></item>
+ /// <item><term>trec.doc.parser</term><description>specifies the <see cref="TrecDocParser"/> class to use for
+ /// parsing the TREC documents content (<b>default=TrecGov2Parser</b>).
+ /// </description></item>
+ /// <item><term>html.parser</term><description>specifies the <see cref="IHTMLParser"/> class to use for
+ /// parsing the HTML parts of the TREC documents content (<b>default=DemoHTMLParser</b>).
+ /// </description></item>
+ /// <item><term>content.source.encoding</term><description>if not specified, ISO-8859-1 is used.</description></item>
+ /// <item>content.source.excludeIteration<term></term><description>if <c>true</c>, do not append iteration number to docname</description></item>
+ /// </list>
+ /// </remarks>
+ public class TrecContentSource : ContentSource
+ {
+ // LUCENENET specific - DateFormatInfo not used
+
+ public static readonly string DOCNO = "<DOCNO>";
+ public static readonly string TERMINATING_DOCNO = "</DOCNO>";
+ public static readonly string DOC = "<DOC>";
+ public static readonly string TERMINATING_DOC = "</DOC>";
+
+ /// <summary>separator between lines in the buffer</summary>
+ public static readonly string NEW_LINE = Environment.NewLine;
+
+ private static readonly string[] DATE_FORMATS = {
+ // LUCENENET specific: in JAVA, they don't care if it is an abbreviated or a full month name when parsing
+ // so we provide definitions for both ways.
+ "ddd, dd MMM yyyy hh:mm:ss K", // Tue, 09 Dec 2003 22:39:08 GMT
+ "ddd, dd MMMM yyyy hh:mm:ss K", // Tue, 09 December 2003 22:39:08 GMT
+ "ddd MMM dd hh:mm:ss yyyy K", // Tue Dec 09 16:45:08 2003 EST
+ "ddd MMMM dd hh:mm:ss yyyy K", // Tue December 09 16:45:08 2003 EST
+ "ddd, dd-MMM-':'y hh:mm:ss K", // Tue, 09 Dec 2003 22:39:08 GMT
+ "ddd, dd-MMMM-':'y hh:mm:ss K", // Tue, 09 December 2003 22:39:08 GMT
+ "ddd, dd-MMM-yyy hh:mm:ss K", // Tue, 09 Dec 2003 22:39:08 GMT
+ "ddd, dd-MMMM-yyy hh:mm:ss K", // Tue, 09 December 2003 22:39:08 GMT
+ "ddd MMM dd hh:mm:ss yyyy", // Tue Dec 09 16:45:08 2003
+ "ddd MMMM dd hh:mm:ss yyyy", // Tue December 09 16:45:08 2003
+ "dd MMM yyyy", // 1 Mar 1994
+ "dd MMMM yyyy", // 1 March 1994
+ "MMM dd, yyyy", // Feb 3, 1994
+ "MMMM dd, yyyy", // February 3, 1994
+ "yyMMdd", // 910513
+ "hhmm K.K.K. MMM dd, yyyy", // 0901 u.t.c. Apr 28, 1994
+ "hhmm K.K.K. MMMM dd, yyyy", // 0901 u.t.c. April 28, 1994
+ };
+
+ private ThreadLocal<StringBuilder> trecDocBuffer = new ThreadLocal<StringBuilder>();
+ private DirectoryInfo dataDir = null;
+ private List<FileInfo> inputFiles = new List<FileInfo>();
+ private int nextFile = 0;
+ // Use to synchronize threads on reading from the TREC documents.
+ private object @lock = new object();
+
+ // Required for test
+ internal TextReader reader;
+ internal int iteration = 0;
+ internal IHTMLParser htmlParser;
+
+ private bool excludeDocnameIteration;
+ private TrecDocParser trecDocParser = new TrecGov2Parser(); // default
+ internal TrecDocParser.ParsePathType currPathType; // not private for tests
+
+ private StringBuilder GetDocBuffer()
+ {
+ StringBuilder sb = trecDocBuffer.Value;
+ if (sb == null)
+ {
+ sb = new StringBuilder();
+ trecDocBuffer.Value = sb;
+ }
+ return sb;
+ }
+
+ internal IHTMLParser HtmlParser
+ {
+ get { return htmlParser; }
+ }
+
+ /// <summary>
+ /// Read until a line starting with the specified <paramref name="lineStart"/>.
+ /// </summary>
+ /// <param name="buf">Buffer for collecting the data if so specified.</param>
+ /// <param name="lineStart">Line start to look for, must not be <c>null</c>.</param>
+ /// <param name="collectMatchLine">Whether to collect the matching line into <c>buffer</c>.</param>
+ /// <param name="collectAll">Whether to collect all lines into <c>buffer</c>.</param>
+ /// <exception cref="IOException">If there is a low-level I/O error.</exception>
+ /// <exception cref="NoMoreDataException">If the source is exhausted.</exception>
+ private void Read(StringBuilder buf, string lineStart,
+ bool collectMatchLine, bool collectAll)
+ {
+ string sep = "";
+ while (true)
+ {
+ string line = reader.ReadLine();
+
+ if (line == null)
+ {
+ OpenNextFile();
+ continue;
+ }
+
+ var _ = line.Length;
+
+ if (lineStart != null && line.StartsWith(lineStart, StringComparison.Ordinal))
+ {
+ if (collectMatchLine)
+ {
+ buf.Append(sep).Append(line);
+ sep = NEW_LINE;
+ }
+ return;
+ }
+
+ if (collectAll)
+ {
+ buf.Append(sep).Append(line);
+ sep = NEW_LINE;
+ }
+ }
+ }
+
+ internal virtual void OpenNextFile()
+ {
+ Dispose();
+ //currPathType = null;
+ while (true)
+ {
+ if (nextFile >= inputFiles.Count)
+ {
+ // exhausted files, start a new round, unless forever set to false.
+ if (!m_forever)
+ {
+ throw new NoMoreDataException();
+ }
+ nextFile = 0;
+ iteration++;
+ }
+ FileInfo f = inputFiles[nextFile++];
+ if (m_verbose)
+ {
+ SystemConsole.WriteLine("opening: " + f + " length: " + f.Length);
+ }
+ try
+ {
+ Stream inputStream = StreamUtils.GetInputStream(f); // support either gzip, bzip2, or regular text file, by extension
+ reader = new StreamReader(inputStream, m_encoding);
+ currPathType = TrecDocParser.PathType(f);
+ return;
+ }
+ catch (Exception e)
+ {
+ if (m_verbose)
+ {
+ SystemConsole.WriteLine("Skipping 'bad' file " + f.FullName + " due to " + e.Message);
+ continue;
+ }
+ throw new NoMoreDataException();
+ }
+ }
+ }
+
+ public virtual DateTime? ParseDate(string dateStr)
+ {
+ dateStr = dateStr.Trim();
+ DateTime d;
+ if (DateTime.TryParseExact(dateStr, DATE_FORMATS, CultureInfo.InvariantCulture, DateTimeStyles.None, out d))
+ {
+ return d;
+ }
+ else if (DateTime.TryParse(dateStr, CultureInfo.InvariantCulture, DateTimeStyles.None, out d))
+ {
+ return d;
+ }
+
+ // do not fail test just because a date could not be parsed
+ if (m_verbose)
+ {
+ SystemConsole.WriteLine("failed to parse date (assigning 'now') for: " + dateStr);
+ }
+ return null;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (reader == null)
+ {
+ return;
+ }
+
+ try
+ {
+ reader.Dispose();
+ }
+ catch (IOException e)
+ {
+ if (m_verbose)
+ {
+ SystemConsole.WriteLine("failed to dispose reader !");
+ SystemConsole.WriteLine(e.ToString());
+ }
+ }
+ reader = null;
+ }
+
+ public override DocData GetNextDocData(DocData docData)
+ {
+ string name = null;
+ StringBuilder docBuf = GetDocBuffer();
+ TrecDocParser.ParsePathType parsedPathType;
+
+ // protect reading from the TREC files by multiple threads. The rest of the
+ // method, i.e., parsing the content and returning the DocData can run unprotected.
+ lock (@lock)
+ {
+ if (reader == null)
+ {
+ OpenNextFile();
+ }
+
+ // 1. skip until doc start - required for all TREC formats
+ docBuf.Length = 0;
+ Read(docBuf, DOC, false, false);
+
+ // save parsedFile for passing trecDataParser after the sync block, in
+ // case another thread will open another file in between.
+ parsedPathType = currPathType;
+
+ // 2. name - required for all TREC formats
+ docBuf.Length = 0;
+ Read(docBuf, DOCNO, true, false);
+ name = docBuf.ToString(DOCNO.Length, docBuf.IndexOf(TERMINATING_DOCNO,
+ DOCNO.Length) - DOCNO.Length).Trim();
+
+ if (!excludeDocnameIteration)
+ {
+ name = name + "_" + iteration;
+ }
+
+ // 3. read all until end of doc
+ docBuf.Length = 0;
+ Read(docBuf, TERMINATING_DOC, false, true);
+ }
+
+ // count char length of text to be parsed (may be larger than the resulted plain doc body text).
+ AddBytes(docBuf.Length);
+
+ // This code segment relies on HtmlParser being thread safe. When we get
+ // here, everything else is already private to that thread, so we're safe.
+ docData = trecDocParser.Parse(docData, name, this, docBuf, parsedPathType);
+ AddItem();
+
+ return docData;
+ }
+
+ public override void ResetInputs()
+ {
+ lock (@lock)
+ {
+ base.ResetInputs();
+ Dispose();
+ nextFile = 0;
+ iteration = 0;
+ }
+ }
+
+ public override void SetConfig(Config config)
+ {
+ base.SetConfig(config);
+ // dirs
+ DirectoryInfo workDir = new DirectoryInfo(config.Get("work.dir", "work"));
+ string d = config.Get("docs.dir", "trec");
+ dataDir = new DirectoryInfo(d);
+ // files
+ CollectFiles(dataDir, inputFiles);
+ if (inputFiles.Count == 0)
+ {
+ throw new ArgumentException("No files in dataDir: " + dataDir);
+ }
+ // trec doc parser
+ try
+ {
+ string trecDocParserClassName = config.Get("trec.doc.parser", "Lucene.Net.Benchmarks.ByTask.Feeds.TrecGov2Parser, Lucene.Net.Benchmark");
+ trecDocParser = (TrecDocParser)Activator.CreateInstance(Type.GetType(trecDocParserClassName));
+ }
+ catch (Exception e)
+ {
+ // Should not get here. Throw runtime exception.
+ throw new Exception(e.ToString(), e);
+ }
+ // html parser
+ try
+ {
+ string htmlParserClassName = config.Get("html.parser",
+ "Lucene.Net.Benchmarks.ByTask.Feeds.DemoHTMLParser, Lucene.Net.Benchmark");
+ htmlParser = (IHTMLParser)Activator.CreateInstance(Type.GetType(htmlParserClassName));
+ }
+ catch (Exception e)
+ {
+ // Should not get here. Throw runtime exception.
+ throw new Exception(e.ToString(), e);
+ }
+ // encoding
+ if (m_encoding == null)
+ {
+ m_encoding = Encoding.GetEncoding("iso-8859-1"); //StandardCharsets.ISO_8859_1.name();
+ }
+ // iteration exclusion in doc name
+ excludeDocnameIteration = config.Get("content.source.excludeIteration", false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecDocParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecDocParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecDocParser.cs
new file mode 100644
index 0000000..b67a1c0
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecDocParser.cs
@@ -0,0 +1,159 @@
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for trec doc content, invoked on doc text excluding <DOC> and <DOCNO>
+ /// which are handled in TrecContentSource. Required to be stateless and hence thread safe.
+ /// </summary>
+ public abstract class TrecDocParser
+ {
+ /// <summary>Types of trec parse paths,</summary>
+ public enum ParsePathType { GOV2, FBIS, FT, FR94, LATIMES, UNKNOWN }
+
+ /// <summary>trec parser type used for unknown extensions</summary>
+ public static readonly ParsePathType DEFAULT_PATH_TYPE = ParsePathType.GOV2;
+
+ internal static readonly IDictionary<ParsePathType, TrecDocParser> pathType2parser = new Dictionary<ParsePathType, TrecDocParser>();
+ static TrecDocParser()
+ {
+ pathType2parser[ParsePathType.GOV2] = new TrecGov2Parser();
+ pathType2parser[ParsePathType.FBIS] = new TrecFBISParser();
+ pathType2parser[ParsePathType.FR94] = new TrecFR94Parser();
+ pathType2parser[ParsePathType.FT] = new TrecFTParser();
+ pathType2parser[ParsePathType.LATIMES] = new TrecLATimesParser();
+
+ foreach (ParsePathType ppt in Enum.GetValues(typeof(ParsePathType)))
+ {
+ pathName2Type[ppt.ToString().ToUpperInvariant()] = ppt;
+ }
+ }
+
+ internal static readonly IDictionary<string, ParsePathType?> pathName2Type = new Dictionary<string, ParsePathType?>();
+
+
+ /// <summary>max length of walk up from file to its ancestors when looking for a known path type.</summary>
+ private static readonly int MAX_PATH_LENGTH = 10;
+
+ /// <summary>
+ /// Compute the path type of a file by inspecting name of file and its parents.
+ /// </summary>
+ public static ParsePathType PathType(FileInfo f)
+ {
+ int pathLength = 0;
+ ParsePathType? ppt;
+ if (pathName2Type.TryGetValue(f.Name.ToUpperInvariant(), out ppt) && ppt != null)
+ {
+ return ppt.Value;
+ }
+ // Walk up the directory names to find a match.
+ DirectoryInfo parentDir = f.Directory;
+ while (parentDir != null && ++pathLength < MAX_PATH_LENGTH)
+ {
+ if (pathName2Type.TryGetValue(parentDir.Name.ToUpperInvariant(), out ppt) && ppt != null)
+ {
+ return ppt.Value;
+ }
+ parentDir = parentDir.Parent;
+ }
+ return DEFAULT_PATH_TYPE;
+ }
+
+ /// <summary>
+ /// Parse the text prepared in docBuf into a result DocData,
+ /// no synchronization is required.
+ /// </summary>
+ /// <param name="docData">Reusable result.</param>
+ /// <param name="name">Name that should be set to the result.</param>
+ /// <param name="trecSrc">Calling trec content source.</param>
+ /// <param name="docBuf">Text to parse.</param>
+ /// <param name="pathType">Type of parsed file, or <see cref="ParsePathType.UNKNOWN"/> if unknown - may be used by
+ /// parsers to alter their behavior according to the file path type. </param>
+ /// <returns></returns>
+ public abstract DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType);
+
+ /// <summary>
+ /// strip tags from <code>buf</code>: each tag is replaced by a single blank.
+ /// </summary>
+ /// <returns>Text obtained when stripping all tags from <paramref name="buf"/> (input <see cref="StringBuilder"/> is unmodified).</returns>
+ public static string StripTags(StringBuilder buf, int start)
+ {
+ return StripTags(buf.ToString(start, buf.Length - start), 0);
+ }
+
+ /// <summary>
+ /// Strip tags from input.
+ /// </summary>
+ /// <seealso cref="StripTags(StringBuilder, int)"/>
+ public static string StripTags(string buf, int start)
+ {
+ if (start > 0)
+ {
+ buf = buf.Substring(0);
+ }
+ return Regex.Replace(buf, "<[^>]*>", " ");
+ }
+
+ /// <summary>
+ /// Extract from <paramref name="buf"/> the text of interest within specified tags.
+ /// </summary>
+ /// <param name="buf">Entire input text.</param>
+ /// <param name="startTag">Tag marking start of text of interest.</param>
+ /// <param name="endTag">Tag marking end of text of interest.</param>
+ /// <param name="maxPos">if ≥ 0 sets a limit on start of text of interest.</param>
+ /// <param name="noisePrefixes">Text of interest or null if not found.</param>
+ /// <returns></returns>
+ public static string Extract(StringBuilder buf, string startTag, string endTag, int maxPos, string[] noisePrefixes)
+ {
+ int k1 = buf.IndexOf(startTag);
+ if (k1 >= 0 && (maxPos < 0 || k1 < maxPos))
+ {
+ k1 += startTag.Length;
+ int k2 = buf.IndexOf(endTag, k1);
+ if (k2 >= 0 && (maxPos < 0 || k2 < maxPos))
+ { // found end tag with allowed range
+ if (noisePrefixes != null)
+ {
+ foreach (string noise in noisePrefixes)
+ {
+ int k1a = buf.IndexOf(noise, k1);
+ if (k1a >= 0 && k1a < k2)
+ {
+ k1 = k1a + noise.Length;
+ }
+ }
+ }
+ return buf.ToString(k1, k2 - k1).Trim();
+ }
+ }
+ return null;
+ }
+
+ //public static void main(String[] args) {
+ // System.out.println(stripTags("is it true that<space>2<<second space>><almost last space>1<one more space>?",0));
+ //}
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFBISParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFBISParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFBISParser.cs
new file mode 100644
index 0000000..cf321cc
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFBISParser.cs
@@ -0,0 +1,68 @@
+using Lucene.Net.Support;
+using System;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for the FBIS docs in trec disks 4+5 collection format
+ /// </summary>
+ public class TrecFBISParser : TrecDocParser
+ {
+ private static readonly string HEADER = "<HEADER>";
+ private static readonly string HEADER_END = "</HEADER>";
+ private static readonly int HEADER_END_LENGTH = HEADER_END.Length;
+
+ private static readonly string DATE1 = "<DATE1>";
+ private static readonly string DATE1_END = "</DATE1>";
+
+ private static readonly string TI = "<TI>";
+ private static readonly string TI_END = "</TI>";
+
+ public override DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType)
+ {
+ int mark = 0; // that much is skipped
+ // optionally skip some of the text, set date, title
+ DateTime? date = null;
+ string title = null;
+ int h1 = docBuf.IndexOf(HEADER);
+ if (h1 >= 0)
+ {
+ int h2 = docBuf.IndexOf(HEADER_END, h1);
+ mark = h2 + HEADER_END_LENGTH;
+ // date...
+ string dateStr = Extract(docBuf, DATE1, DATE1_END, h2, null);
+ if (dateStr != null)
+ {
+ date = trecSrc.ParseDate(dateStr);
+ }
+ // title...
+ title = Extract(docBuf, TI, TI_END, h2, null);
+ }
+ docData.Clear();
+ docData.Name = name;
+ docData.SetDate(date);
+ docData.Title = title;
+ docData.Body = StripTags(docBuf, mark).ToString();
+ return docData;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFR94Parser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFR94Parser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFR94Parser.cs
new file mode 100644
index 0000000..72f99bb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFR94Parser.cs
@@ -0,0 +1,69 @@
+using Lucene.Net.Support;
+using System;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for the FR94 docs in trec disks 4+5 collection format
+ /// </summary>
+ public class TrecFR94Parser : TrecDocParser
+ {
+ private static readonly string TEXT = "<TEXT>";
+ private static readonly int TEXT_LENGTH = TEXT.Length;
+ private static readonly string TEXT_END = "</TEXT>";
+
+ private static readonly string DATE = "<DATE>";
+ private static readonly string[] DATE_NOISE_PREFIXES = {
+ "DATE:",
+ "date:", //TODO improve date extraction for this format
+ "t.c.",
+ };
+ private static readonly string DATE_END = "</DATE>";
+
+ //TODO can we also extract title for this format?
+
+ public override DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType)
+ {
+ int mark = 0; // that much is skipped
+ // optionally skip some of the text, set date (no title?)
+ DateTime? date = null;
+ int h1 = docBuf.IndexOf(TEXT);
+ if (h1 >= 0)
+ {
+ int h2 = docBuf.IndexOf(TEXT_END, h1);
+ mark = h1 + TEXT_LENGTH;
+ // date...
+ string dateStr = Extract(docBuf, DATE, DATE_END, h2, DATE_NOISE_PREFIXES);
+ if (dateStr != null)
+ {
+ dateStr = StripTags(dateStr, 0).ToString();
+ date = trecSrc.ParseDate(dateStr.Trim());
+ }
+ }
+ docData.Clear();
+ docData.Name = name;
+ docData.SetDate(date);
+ docData.Body = StripTags(docBuf, mark).ToString();
+ return docData;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFTParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFTParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFTParser.cs
new file mode 100644
index 0000000..189f6cb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecFTParser.cs
@@ -0,0 +1,58 @@
+using System;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for the FT docs in trec disks 4+5 collection format
+ /// </summary>
+ public class TrecFTParser : TrecDocParser
+ {
+ private static readonly string DATE = "<DATE>";
+ private static readonly string DATE_END = "</DATE>";
+
+ private static readonly string HEADLINE = "<HEADLINE>";
+ private static readonly string HEADLINE_END = "</HEADLINE>";
+
+ public override DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType)
+ {
+ int mark = 0; // that much is skipped
+
+ // date...
+ DateTime? date = null;
+ string dateStr = Extract(docBuf, DATE, DATE_END, -1, null);
+ if (dateStr != null)
+ {
+ date = trecSrc.ParseDate(dateStr);
+ }
+
+ // title...
+ string title = Extract(docBuf, HEADLINE, HEADLINE_END, -1, null);
+
+ docData.Clear();
+ docData.Name = name;
+ docData.SetDate(date);
+ docData.Title = title;
+ docData.Body = StripTags(docBuf, mark).ToString();
+ return docData;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecGov2Parser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecGov2Parser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecGov2Parser.cs
new file mode 100644
index 0000000..12912e9
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecGov2Parser.cs
@@ -0,0 +1,57 @@
+using Lucene.Net.Support;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for the GOV2 collection format
+ /// </summary>
+ public class TrecGov2Parser : TrecDocParser
+ {
+ private static readonly string DATE = "Date: ";
+ private static readonly string DATE_END = TrecContentSource.NEW_LINE;
+
+ private static readonly string DOCHDR = "<DOCHDR>";
+ private static readonly string TERMINATING_DOCHDR = "</DOCHDR>";
+
+ public override DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType)
+ {
+ // skip some of the non-html text, optionally set date
+ DateTime? date = null;
+ int start = 0;
+ int h1 = docBuf.IndexOf(DOCHDR);
+ if (h1 >= 0)
+ {
+ int h2 = docBuf.IndexOf(TERMINATING_DOCHDR, h1);
+ string dateStr = Extract(docBuf, DATE, DATE_END, h2, null);
+ if (dateStr != null)
+ {
+ date = trecSrc.ParseDate(dateStr);
+ }
+ start = h2 + TERMINATING_DOCHDR.Length;
+ }
+ string html = docBuf.ToString(start, docBuf.Length - start);
+ return trecSrc.HtmlParser.Parse(docData, name, date, new StringReader(html), trecSrc);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecLATimesParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecLATimesParser.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecLATimesParser.cs
new file mode 100644
index 0000000..e54f635
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecLATimesParser.cs
@@ -0,0 +1,75 @@
+using System;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for the FT docs in trec disks 4+5 collection format
+ /// </summary>
+ public class TrecLATimesParser : TrecDocParser
+ {
+ private static readonly string DATE = "<DATE>";
+ private static readonly string DATE_END = "</DATE>";
+ private static readonly string DATE_NOISE = "day,"; // anything aftre the ','
+
+ private static readonly string SUBJECT = "<SUBJECT>";
+ private static readonly string SUBJECT_END = "</SUBJECT>";
+ private static readonly string HEADLINE = "<HEADLINE>";
+ private static readonly string HEADLINE_END = "</HEADLINE>";
+
+ public override DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType)
+ {
+ int mark = 0; // that much is skipped
+
+ // date...
+ DateTime? date = null;
+ string dateStr = Extract(docBuf, DATE, DATE_END, -1, null);
+ if (dateStr != null)
+ {
+ int d2a = dateStr.IndexOf(DATE_NOISE);
+ if (d2a > 0)
+ {
+ dateStr = dateStr.Substring(0, (d2a + 3) - 0); // we need the "day" part
+ }
+ dateStr = StripTags(dateStr, 0).ToString();
+ date = trecSrc.ParseDate(dateStr.Trim());
+ }
+
+ // title... first try with SUBJECT, them with HEADLINE
+ string title = Extract(docBuf, SUBJECT, SUBJECT_END, -1, null);
+ if (title == null)
+ {
+ title = Extract(docBuf, HEADLINE, HEADLINE_END, -1, null);
+ }
+ if (title != null)
+ {
+ title = StripTags(title, 0).ToString().Trim();
+ }
+
+ docData.Clear();
+ docData.Name = name;
+ docData.SetDate(date);
+ docData.Title = title;
+ docData.Body = StripTags(docBuf, mark).ToString();
+ return docData;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecParserByPath.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecParserByPath.cs b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecParserByPath.cs
new file mode 100644
index 0000000..45a72b4
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Feeds/TrecParserByPath.cs
@@ -0,0 +1,34 @@
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Parser for trec docs which selects the parser to apply according
+ /// to the source files path, defaulting to <see cref="TrecGov2Parser"/>.
+ /// </summary>
+ public class TrecParserByPath : TrecDocParser
+ {
+ public override DocData Parse(DocData docData, string name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType)
+ {
+ return pathType2parser[pathType].Parse(docData, name, trecSrc, docBuf, pathType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/PerfRunData.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/PerfRunData.cs b/src/Lucene.Net.Benchmark/ByTask/PerfRunData.cs
new file mode 100644
index 0000000..e5b334c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/PerfRunData.cs
@@ -0,0 +1,490 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Facet.Taxonomy;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Data maintained by a performance test run.
+ /// </summary>
+ /// <remarks>
+ /// Data includes:
+ /// <list type="bullet">
+ /// <item><description>Configuration.</description></item>
+ /// <item><description>Directory, Writer, Reader.</description></item>
+ /// <item><description>Taxonomy Directory, Writer, Reader.</description></item>
+ /// <item><description>DocMaker, FacetSource and a few instances of QueryMaker.</description></item>
+ /// <item><description>Named AnalysisFactories.</description></item>
+ /// <item><description>Analyzer.</description></item>
+ /// <item><description>Statistics data which updated during the run.</description></item>
+ /// </list>
+ /// <para/>
+ /// Config properties:
+ /// <list type="bullet">
+ /// <item><term>work.dir</term><description><path to root of docs and index dirs| Default: work></description></item>
+ /// <item><term>analyzer</term><description><class name for analyzer| Default: StandardAnalyzer></description></item>
+ /// <item><term>doc.maker</term><description><class name for doc-maker| Default: DocMaker></description></item>
+ /// <item><term>facet.source</term><description><class name for facet-source| Default: RandomFacetSource></description></item>
+ /// <item><term>query.maker</term><description><class name for query-maker| Default: SimpleQueryMaker></description></item>
+ /// <item><term>log.queries</term><description><whether queries should be printed| Default: false></description></item>
+ /// <item><term>directory</term><description><type of directory to use for the index| Default: RAMDirectory></description></item>
+ /// <item><term>taxonomy.directory</term><description><type of directory for taxonomy index| Default: RAMDirectory></description></item>
+ /// </list>
+ /// </remarks>
+ public class PerfRunData : IDisposable
+ {
+ private Points points;
+
+ // objects used during performance test run
+ // directory, analyzer, docMaker - created at startup.
+ // reader, writer, searcher - maintained by basic tasks.
+ private Store.Directory directory;
+ private IDictionary<string, AnalyzerFactory> analyzerFactories = new Dictionary<string, AnalyzerFactory>();
+ private Analyzer analyzer;
+ private DocMaker docMaker;
+ private ContentSource contentSource;
+ private FacetSource facetSource;
+ private CultureInfo locale;
+
+ private Store.Directory taxonomyDir;
+ private ITaxonomyWriter taxonomyWriter;
+ private TaxonomyReader taxonomyReader;
+
+ // we use separate (identical) instances for each "read" task type, so each can iterate the quries separately.
+ private IDictionary<Type, IQueryMaker> readTaskQueryMaker;
+ private Type qmkrClass;
+
+ private DirectoryReader indexReader;
+ private IndexSearcher indexSearcher;
+ private IndexWriter indexWriter;
+ private Config config;
+ private long startTimeMillis;
+
+ private readonly IDictionary<string, object> perfObjects = new Dictionary<string, object>();
+
+ // constructor
+ public PerfRunData(Config config)
+ {
+ this.config = config;
+ // analyzer (default is standard analyzer)
+ analyzer = NewAnalyzerTask.CreateAnalyzer(config.Get("analyzer",
+ "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common"));
+
+ // content source
+ string sourceClass = config.Get("content.source", typeof(SingleDocSource).AssemblyQualifiedName);
+ contentSource = (ContentSource)Activator.CreateInstance(Type.GetType(sourceClass)); //Class.forName(sourceClass).asSubclass(typeof(ContentSource)).newInstance();
+ contentSource.SetConfig(config);
+
+ // doc maker
+ docMaker = (DocMaker)Activator.CreateInstance(Type.GetType(config.Get("doc.maker", typeof(DocMaker).AssemblyQualifiedName))); // "org.apache.lucene.benchmark.byTask.feeds.DocMaker")).asSubclass(DocMaker.class).newInstance();
+ docMaker.SetConfig(config, contentSource);
+ // facet source
+ facetSource = (FacetSource)Activator.CreateInstance(Type.GetType(config.Get("facet.source",
+ typeof(RandomFacetSource).AssemblyQualifiedName))); // "org.apache.lucene.benchmark.byTask.feeds.RandomFacetSource")).asSubclass(FacetSource.class).newInstance();
+ facetSource.SetConfig(config);
+ // query makers
+ readTaskQueryMaker = new Dictionary<Type, IQueryMaker>();
+ qmkrClass = Type.GetType(config.Get("query.maker", typeof(SimpleQueryMaker).AssemblyQualifiedName));
+
+ // index stuff
+ Reinit(false);
+
+ // statistic points
+ points = new Points(config);
+
+ if (bool.Parse(config.Get("log.queries", "false")))
+ {
+ SystemConsole.WriteLine("------------> queries:");
+ SystemConsole.WriteLine(GetQueryMaker(new SearchTask(this)).PrintQueries());
+ }
+ }
+
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ IOUtils.Dispose(indexWriter, indexReader, directory,
+ taxonomyWriter, taxonomyReader, taxonomyDir,
+ docMaker, facetSource, contentSource);
+
+ // close all perf objects that are closeable.
+ List<IDisposable> perfObjectsToClose = new List<IDisposable>();
+ foreach (object obj in perfObjects.Values)
+ {
+ if (obj is IDisposable)
+ {
+ perfObjectsToClose.Add((IDisposable)obj);
+ }
+ }
+ IOUtils.Dispose(perfObjectsToClose);
+ }
+ }
+
+ // clean old stuff, reopen
+ public virtual void Reinit(bool eraseIndex)
+ {
+ // cleanup index
+ IOUtils.Dispose(indexWriter, indexReader, directory);
+ indexWriter = null;
+ indexReader = null;
+
+ IOUtils.Dispose(taxonomyWriter, taxonomyReader, taxonomyDir);
+ taxonomyWriter = null;
+ taxonomyReader = null;
+
+ // directory (default is ram-dir).
+ directory = CreateDirectory(eraseIndex, "index", "directory");
+ taxonomyDir = CreateDirectory(eraseIndex, "taxo", "taxonomy.directory");
+
+ // inputs
+ ResetInputs();
+
+ // release unused stuff
+ GC.Collect();
+
+ // Re-init clock
+ SetStartTimeMillis();
+ }
+
+ private Store.Directory CreateDirectory(bool eraseIndex, string dirName,
+ string dirParam)
+ {
+ if ("FSDirectory".Equals(config.Get(dirParam, "RAMDirectory"), StringComparison.Ordinal))
+ {
+ DirectoryInfo workDir = new DirectoryInfo(config.Get("work.dir", "work"));
+ DirectoryInfo indexDir = new DirectoryInfo(System.IO.Path.Combine(workDir.FullName, dirName));
+ if (eraseIndex && indexDir.Exists)
+ {
+ FileUtils.FullyDelete(indexDir);
+ }
+ indexDir.Create();
+ return FSDirectory.Open(indexDir);
+ }
+
+ return new RAMDirectory();
+ }
+
+ /// <summary>
+ /// Returns an object that was previously set by <see cref="SetPerfObject(string, object)"/>.
+ /// </summary>
+ public virtual object GetPerfObject(string key)
+ {
+ lock (this)
+ {
+ object result;
+ perfObjects.TryGetValue(key, out result);
+ return result;
+ }
+ }
+
+ /// <summary>
+ /// Sets an object that is required by <see cref="PerfTask"/>s, keyed by the given
+ /// <paramref name="key"/>. If the object implements <see cref="IDisposable"/>, it will be disposed
+ /// by <see cref="Dispose()"/>.
+ /// </summary>
+ public virtual void SetPerfObject(string key, object obj)
+ {
+ lock (this)
+ {
+ perfObjects[key] = obj;
+ }
+ }
+
+ public virtual long SetStartTimeMillis()
+ {
+ startTimeMillis = Support.Time.CurrentTimeMilliseconds();
+ return startTimeMillis;
+ }
+
+ /// <summary>
+ /// Gets start time in milliseconds.
+ /// </summary>
+ public virtual long StartTimeMillis
+ {
+ get { return startTimeMillis; }
+ }
+
+ /// <summary>
+ /// Gets the points.
+ /// </summary>
+ public virtual Points Points
+ {
+ get { return points; }
+ }
+
+ /// <summary>
+ /// Gets or sets the directory.
+ /// </summary>
+ public virtual Store.Directory Directory
+ {
+ get { return directory; }
+ set { directory = value; }
+ }
+
+ /// <summary>
+ /// Gets the taxonomy directory.
+ /// </summary>
+ public virtual Store.Directory TaxonomyDir
+ {
+ get { return taxonomyDir; }
+ }
+
+ /// <summary>
+ /// Set the taxonomy reader. Takes ownership of that taxonomy reader, that is,
+ /// internally performs taxoReader.IncRef() (If caller no longer needs that
+ /// reader it should DecRef()/Dispose() it after calling this method, otherwise,
+ /// the reader will remain open).
+ /// </summary>
+ /// <param name="taxoReader">The taxonomy reader to set.</param>
+ public virtual void SetTaxonomyReader(TaxonomyReader taxoReader)
+ {
+ lock (this)
+ {
+ if (taxoReader == this.taxonomyReader)
+ {
+ return;
+ }
+ if (taxonomyReader != null)
+ {
+ taxonomyReader.DecRef();
+ }
+
+ if (taxoReader != null)
+ {
+ taxoReader.IncRef();
+ }
+ this.taxonomyReader = taxoReader;
+ }
+ }
+
+ /// <summary>
+ /// Returns the taxonomyReader. NOTE: this returns a
+ /// reference. You must call TaxonomyReader.DecRef() when
+ /// you're done.
+ /// </summary>
+ public virtual TaxonomyReader GetTaxonomyReader()
+ {
+ lock (this)
+ {
+ if (taxonomyReader != null)
+ {
+ taxonomyReader.IncRef();
+ }
+ return taxonomyReader;
+ }
+ }
+
+ /// <summary>
+ /// Gets or sets the taxonomy writer.
+ /// </summary>
+ public virtual ITaxonomyWriter TaxonomyWriter
+ {
+ get { return taxonomyWriter; }
+ set { taxonomyWriter = value; }
+ }
+
+ /// <summary>
+ /// Returns the indexReader. NOTE: this returns a
+ /// reference. You must call IndexReader.DecRef() when
+ /// you're done.
+ /// </summary>
+ public virtual DirectoryReader GetIndexReader()
+ {
+ lock (this)
+ {
+ if (indexReader != null)
+ {
+ indexReader.IncRef();
+ }
+ return indexReader;
+ }
+ }
+
+ /// <summary>
+ /// Returns the indexSearcher. NOTE: this returns
+ /// a reference to the underlying IndexReader. You must
+ /// call IndexReader.DecRef() when you're done.
+ /// </summary>
+ /// <returns></returns>
+ public virtual IndexSearcher GetIndexSearcher()
+ {
+ lock (this)
+ {
+ if (indexReader != null)
+ {
+ indexReader.IncRef();
+ }
+ return indexSearcher;
+ }
+ }
+
+ /// <summary>
+ /// Set the index reader. Takes ownership of that index reader, that is,
+ /// internally performs indexReader.incRef() (If caller no longer needs that
+ /// reader it should decRef()/close() it after calling this method, otherwise,
+ /// the reader will remain open).
+ /// </summary>
+ /// <param name="indexReader">The indexReader to set.</param>
+ public virtual void SetIndexReader(DirectoryReader indexReader)
+ {
+ lock (this)
+ {
+ if (indexReader == this.indexReader)
+ {
+ return;
+ }
+
+ if (this.indexReader != null)
+ {
+ // Release current IR
+ this.indexReader.DecRef();
+ }
+
+ this.indexReader = indexReader;
+ if (indexReader != null)
+ {
+ // Hold reference to new IR
+ indexReader.IncRef();
+ indexSearcher = new IndexSearcher(indexReader);
+ }
+ else
+ {
+ indexSearcher = null;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Gets or sets the indexWriter.
+ /// </summary>
+ public virtual IndexWriter IndexWriter
+ {
+ get { return indexWriter; }
+ set { indexWriter = value; }
+ }
+
+ /// <summary>
+ /// Gets or sets the analyzer.
+ /// </summary>
+ public virtual Analyzer Analyzer
+ {
+ get { return analyzer; }
+ set { analyzer = value; }
+ }
+
+ /// <summary>Gets the <see cref="Feeds.ContentSource"/>.</summary>
+ public virtual ContentSource ContentSource
+ {
+ get { return contentSource; }
+ }
+
+ /// <summary>Returns the <see cref="Feeds.DocMaker"/>.</summary>
+ public virtual DocMaker DocMaker
+ {
+ get { return docMaker; }
+ }
+
+ /// <summary>Gets the <see cref="Feeds.FacetSource"/>.</summary>
+ public virtual FacetSource FacetSource
+ {
+ get { return facetSource; }
+ }
+
+ /// <summary>
+ /// Gets or sets the culture.
+ /// </summary>
+ public virtual CultureInfo Locale // LUCENENET TODO: API Is this really needed since we have on the thread already?
+ {
+ get { return locale; }
+ set { locale = value; }
+ }
+
+ /// <summary>
+ /// Gets the config.
+ /// </summary>
+ public virtual Config Config
+ {
+ get { return config; }
+ }
+
+ public virtual void ResetInputs()
+ {
+ contentSource.ResetInputs();
+ docMaker.ResetInputs();
+ facetSource.ResetInputs();
+ foreach (IQueryMaker queryMaker in readTaskQueryMaker.Values)
+ {
+ queryMaker.ResetInputs();
+ }
+ }
+
+ /// <summary>
+ /// Returns the queryMaker by read task type (class).
+ /// </summary>
+ public virtual IQueryMaker GetQueryMaker(ReadTask readTask)
+ {
+ lock (this)
+ {
+ // mapping the query maker by task class allows extending/adding new search/read tasks
+ // without needing to modify this class.
+ Type readTaskClass = readTask.GetType();
+ IQueryMaker qm;
+ if (!readTaskQueryMaker.TryGetValue(readTaskClass, out qm) || qm == null)
+ {
+ try
+ {
+ //qm = qmkrClass.newInstance();
+ qm = (IQueryMaker)Activator.CreateInstance(qmkrClass);
+ qm.SetConfig(config);
+ }
+ catch (Exception e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ readTaskQueryMaker[readTaskClass] = qm;
+ }
+ return qm;
+ }
+ }
+
+ public virtual IDictionary<string, AnalyzerFactory> AnalyzerFactories
+ {
+ get { return analyzerFactories; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs b/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
new file mode 100644
index 0000000..6b248f6
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
@@ -0,0 +1,90 @@
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Programmatic
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Sample performance test written programmatically - no algorithm file is needed here.
+ /// </summary>
+ public class Sample
+ {
+ public static void Main(string[] args)
+ {
+ var p = InitProps();
+ Config conf = new Config(p);
+ PerfRunData runData = new PerfRunData(conf);
+
+ // 1. top sequence
+ TaskSequence top = new TaskSequence(runData, null, null, false); // top level, not parallel
+
+ // 2. task to create the index
+ CreateIndexTask create = new CreateIndexTask(runData);
+ top.AddTask(create);
+
+ // 3. task seq to add 500 docs (order matters - top to bottom - add seq to top, only then add to seq)
+ TaskSequence seq1 = new TaskSequence(runData, "AddDocs", top, false);
+ seq1.SetRepetitions(500);
+ seq1.SetNoChildReport();
+ top.AddTask(seq1);
+
+ // 4. task to add the doc
+ AddDocTask addDoc = new AddDocTask(runData);
+ //addDoc.setParams("1200"); // doc size limit if supported
+ seq1.AddTask(addDoc); // order matters 9see comment above)
+
+ // 5. task to close the index
+ CloseIndexTask close = new CloseIndexTask(runData);
+ top.AddTask(close);
+
+ // task to report
+ RepSumByNameTask rep = new RepSumByNameTask(runData);
+ top.AddTask(rep);
+
+ // print algorithm
+ SystemConsole.WriteLine(top.ToString());
+
+ // execute
+ top.DoLogic();
+ }
+
+ // Sample programmatic settings. Could also read from file.
+ private static IDictionary<string, string> InitProps()
+ {
+ var p = new Dictionary<string, string>();
+ p["task.max.depth.log"] = "3";
+ p["max.buffered"] = "buf:10:10:100:100:10:10:100:100";
+ p["doc.maker"] = "Lucene.Net.Benchmarks.ByTask.Feeds.ReutersContentSource, Lucene.Net.Benchmark";
+ p["log.step"] = "2000";
+ p["doc.delete.step"] = "8";
+ p["analyzer"] = "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common";
+ p["doc.term.vector"] = "false";
+ p["directory"] = "FSDirectory";
+ p["query.maker"] = "Lucene.Net.Benchmarks.ByTask.Feeds.ReutersQueryMaker, Lucene.Net.Benchmark";
+ p["doc.stored"] = "true";
+ p["docs.dir"] = "reuters-out";
+ p["compound"] = "cmpnd:true:true:true:true:false:false:false:false";
+ p["doc.tokenized"] = "true";
+ p["merge.factor"] = "mrg:10:100:10:100:10:100:10:100";
+ return p;
+ }
+ }
+}
[22/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Helpers/NamespaceSupport.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Helpers/NamespaceSupport.cs b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/NamespaceSupport.cs
new file mode 100644
index 0000000..4034dad
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/NamespaceSupport.cs
@@ -0,0 +1,841 @@
+// NamespaceSupport.java - generic Namespace support for SAX.
+// http://www.saxproject.org
+// Written by David Megginson
+// This class is in the Public Domain. NO WARRANTY!
+// $Id: NamespaceSupport.java,v 1.15 2004/04/26 17:34:35 dmegginson Exp $
+
+using Lucene.Net.Support;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Sax.Helpers
+{
+ /// <summary>
+ /// Encapsulate Namespace logic for use by applications using SAX,
+ /// or internally by SAX drivers.
+ /// <para/>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// See <a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class encapsulates the logic of Namespace processing: it
+ /// tracks the declarations currently in force for each context and
+ /// automatically processes qualified XML names into their Namespace
+ /// parts; it can also be used in reverse for generating XML qnames
+ /// from Namespaces.
+ /// <para/>
+ /// Namespace support objects are reusable, but the reset method
+ /// must be invoked between each session.
+ /// <para/>Here is a simple session:
+ /// <code>
+ /// string parts[] = new string[3];
+ /// NamespaceSupport support = new NamespaceSupport();
+ /// support.PushContext();
+ /// support.DeclarePrefix("", "http://www.w3.org/1999/xhtml");
+ /// support.DeclarePrefix("dc", "http://www.purl.org/dc#");
+ /// parts = support.ProcessName("p", parts, false);
+ /// Console.WriteLine("Namespace URI: " + parts[0]);
+ /// Console.WriteLine("Local name: " + parts[1]);
+ /// Console.WriteLine("Raw name: " + parts[2]);
+ /// parts = support.ProcessName("dc:title", parts, false);
+ /// Console.WriteLine("Namespace URI: " + parts[0]);
+ /// Console.WriteLine("Local name: " + parts[1]);
+ /// Console.WriteLine("Raw name: " + parts[2]);
+ /// support.PopContext();
+ /// </code>
+ /// <para/>
+ /// Note that this class is optimized for the use case where most
+ /// elements do not contain Namespace declarations: if the same
+ /// prefix/URI mapping is repeated for each context (for example), this
+ /// class will be somewhat less efficient.
+ /// <para/>
+ /// Although SAX drivers (parsers) may choose to use this class to
+ /// implement namespace handling, they are not required to do so.
+ /// Applications must track namespace information themselves if they
+ /// want to use namespace information.
+ /// </summary>
+ public class NamespaceSupport
+ {
+ /// <summary>
+ /// The XML Namespace URI as a constant.
+ /// The value is <c>http://www.w3.org/XML/1998/namespace</c>
+ /// as defined in the "Namespaces in XML" * recommendation.
+ /// <para>This is the Namespace URI that is automatically mapped to the "xml" prefix.</para>
+ /// </summary>
+ public const string XMLNS = "http://www.w3.org/XML/1998/namespace";
+
+ /// <summary>
+ /// The namespace declaration URI as a constant.
+ /// The value is <c>http://www.w3.org/xmlns/2000/</c>, as defined
+ /// in a backwards-incompatible erratum to the "Namespaces in XML"
+ /// recommendation. Because that erratum postdated SAX2, SAX2 defaults
+ /// to the original recommendation, and does not normally use this URI.
+ /// <para/>
+ /// This is the Namespace URI that is optionally applied to
+ /// <em>xmlns</em> and <em>xmlns:*</em> attributes, which are used to
+ /// declare namespaces.
+ /// </summary>
+ /// <seealso cref="SetNamespaceDeclUris(bool)" />
+ /// <seealso cref="IsNamespaceDeclUris" />
+ ////
+ public const string NSDECL = "http://www.w3.org/xmlns/2000/";
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Constructor.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Create a new Namespace support object.
+ /// </summary>
+ public NamespaceSupport()
+ {
+ Reset();
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Context management.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Reset this Namespace support object for reuse.
+ /// <para/>
+ /// It is necessary to invoke this method before reusing the
+ /// Namespace support object for a new session. If namespace
+ /// declaration URIs are to be supported, that flag must also
+ /// be set to a non-default value.
+ /// </summary>
+ /// <seealso cref="SetNamespaceDeclUris" />
+ public void Reset()
+ {
+ contexts = new Context[32];
+ namespaceDeclUris = false;
+ contextPos = 0;
+ contexts[contextPos] = currentContext = new Context(this);
+ currentContext.DeclarePrefix("xml", XMLNS);
+ }
+
+ /// <summary>
+ /// Start a new Namespace context.
+ /// The new context will automatically inherit
+ /// the declarations of its parent context, but it will also keep
+ /// track of which declarations were made within this context.
+ /// <para/>
+ /// Event callback code should start a new context once per element.
+ /// This means being ready to call this in either of two places.
+ /// For elements that don't include namespace declarations, the
+ /// <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/> callback is the right place.
+ /// For elements with such a declaration, it'd done in the first
+ /// <see cref="IContentHandler.StartPrefixMapping(string, string)"/> callback.
+ /// A boolean flag can be used to
+ /// track whether a context has been started yet. When either of
+ /// those methods is called, it checks the flag to see if a new context
+ /// needs to be started. If so, it starts the context and sets the
+ /// flag. After <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/>
+ /// does that, it always clears the flag.
+ /// <para/>
+ /// Normally, SAX drivers would push a new context at the beginning
+ /// of each XML element. Then they perform a first pass over the
+ /// attributes to process all namespace declarations, making
+ /// <see cref="IContentHandler.StartPrefixMapping(string, string)"/> callbacks.
+ /// Then a second pass is made, to determine the namespace-qualified
+ /// names for all attributes and for the element name.
+ /// Finally all the information for the
+ /// <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/> callback is available,
+ /// so it can then be made.
+ /// <para/>
+ /// The Namespace support object always starts with a base context
+ /// already in force: in this context, only the "xml" prefix is
+ /// declared.
+ /// </summary>
+ /// <seealso cref="IContentHandler" />
+ /// <seealso cref="PopContext()" />
+ public void PushContext()
+ {
+ int max = contexts.Length;
+
+ contexts[contextPos].declsOK = false;
+ contextPos++;
+
+ // Extend the array if necessary
+ if (contextPos >= max)
+ {
+ var newContexts = new Context[max * 2];
+ Array.Copy(contexts, 0, newContexts, 0, max);
+ max *= 2;
+ contexts = newContexts;
+ }
+
+ // Allocate the context if necessary.
+ currentContext = contexts[contextPos];
+ if (currentContext == null)
+ {
+ contexts[contextPos] = currentContext = new Context(this);
+ }
+
+ // Set the parent, if any.
+ if (contextPos > 0)
+ {
+ currentContext.SetParent(contexts[contextPos - 1]);
+ }
+ }
+
+ /// <summary>
+ /// Revert to the previous Namespace context.
+ /// <para/>
+ /// Normally, you should pop the context at the end of each
+ /// XML element. After popping the context, all Namespace prefix
+ /// mappings that were previously in force are restored.
+ /// <para/>
+ /// You must not attempt to declare additional Namespace
+ /// prefixes after popping a context, unless you push another
+ /// context first.
+ /// </summary>
+ /// <seealso cref="PushContext()" />
+ public void PopContext()
+ {
+ contexts[contextPos].Clear();
+ contextPos--;
+ if (contextPos < 0)
+ {
+ throw new InvalidOperationException();
+ }
+ currentContext = contexts[contextPos];
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Operations within a context.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Declare a Namespace prefix. All prefixes must be declared
+ /// before they are referenced. For example, a SAX driver (parser)
+ /// would scan an element's attributes
+ /// in two passes: first for namespace declarations,
+ /// then a second pass using <see cref="ProcessName" /> to
+ /// interpret prefixes against (potentially redefined) prefixes.
+ /// <para/>
+ /// This method declares a prefix in the current Namespace
+ /// context; the prefix will remain in force until this context
+ /// is popped, unless it is shadowed in a descendant context.
+ /// <para/>
+ /// To declare the default element Namespace, use the empty string as
+ /// the prefix.
+ /// <para/>
+ /// Note that you must <em>not</em> declare a prefix after
+ /// you've pushed and popped another Namespace context, or
+ /// treated the declarations phase as complete by processing
+ /// a prefixed name.
+ /// <para/>
+ /// Note that there is an asymmetry in this library: <see cref="GetPrefix" /> will not return the "" prefix,
+ /// even if you have declared a default element namespace.
+ /// To check for a default namespace,
+ /// you have to look it up explicitly using <see cref="GetUri" />.
+ /// This asymmetry exists to make it easier to look up prefixes
+ /// for attribute names, where the default prefix is not allowed.
+ /// </summary>
+ /// <param name="prefix">
+ /// The prefix to declare, or the empty string to
+ /// indicate the default element namespace. This may never have
+ /// the value "xml" or "xmlns".
+ /// </param>
+ /// <param name="uri">
+ /// The Namespace URI to associate with the prefix.
+ /// </param>
+ /// <returns><c>true</c> if the prefix was legal, <c>false</c> otherwise</returns>
+ /// <seealso cref="ProcessName(string, string[], bool)" />
+ /// <seealso cref="GetUri(string)" />
+ /// <seealso cref="GetPrefix(string)" />
+ public bool DeclarePrefix(string prefix, string uri)
+ {
+ if (prefix.Equals("xml") || prefix.Equals("xmlns"))
+ {
+ return false;
+ }
+ currentContext.DeclarePrefix(prefix, uri);
+ return true;
+ }
+
+ /// <summary>
+ /// Process a raw XML qualified name, after all declarations in the
+ /// current context have been handled by <see cref="DeclarePrefix" />.
+ /// <para>
+ /// This method processes a raw XML qualified name in the
+ /// current context by removing the prefix and looking it up among
+ /// the prefixes currently declared. The return value will be the
+ /// array supplied by the caller, filled in as follows:
+ /// </para>
+ /// <list type="table">
+ /// <item>
+ /// <term>parts[0]</term>
+ /// <description>The Namespace URI, or an empty string if none is in use.</description>
+ /// </item>
+ /// <item>
+ /// <term>parts[1]</term>
+ /// <description>The local name (without prefix).</description>
+ /// </item>
+ /// <item>
+ /// <term>parts[2]</term>
+ /// <description>The original raw name.</description>
+ /// </item>
+ /// </list>
+ /// <para>
+ /// All of the strings in the array will be internalized. If
+ /// the raw name has a prefix that has not been declared, then
+ /// the return value will be null.
+ /// </para>
+ /// <para>
+ /// Note that attribute names are processed differently than
+ /// element names: an unprefixed element name will receive the
+ /// default Namespace (if any), while an unprefixed attribute name
+ /// will not.
+ /// </para>
+ /// </summary>
+ /// <param name="qName">
+ /// The XML qualified name to be processed.
+ /// </param>
+ /// <param name="parts">
+ /// An array supplied by the caller, capable of
+ /// holding at least three members.
+ /// </param>
+ /// <param name="isAttribute">
+ /// A flag indicating whether this is an
+ /// attribute name (true) or an element name (false).
+ /// </param>
+ /// <returns>
+ /// The supplied array holding three internalized strings
+ /// representing the Namespace URI (or empty string), the
+ /// local name, and the XML qualified name; or null if there
+ /// is an undeclared prefix.
+ /// </returns>
+ /// <seealso cref="DeclarePrefix" />
+ /// <seealso cref="string.Intern(string)" />
+ public string[] ProcessName(string qName, string[] parts, bool isAttribute)
+ {
+ string[] myParts = currentContext.ProcessName(qName, isAttribute);
+ if (myParts == null)
+ {
+ return null;
+ }
+ parts[0] = myParts[0];
+ parts[1] = myParts[1];
+ parts[2] = myParts[2];
+ return parts;
+ }
+
+ /// <summary>
+ /// Look up a prefix and get the currently-mapped Namespace URI.
+ /// <para>
+ /// This method looks up the prefix in the current context.
+ /// Use the empty string ("") for the default Namespace.
+ /// </para>
+ /// </summary>
+ /// <param name="prefix">
+ /// The prefix to look up.
+ /// </param>
+ /// <returns>
+ /// The associated Namespace URI, or null if the prefix
+ /// is undeclared in this context.
+ /// </returns>
+ /// <seealso cref="GetPrefix" />
+ /// <seealso cref="GetPrefixes()" />
+ public string GetUri(string prefix)
+ {
+ return currentContext.GetURI(prefix);
+ }
+
+ /// <summary>
+ /// Return an enumeration of all prefixes whose declarations are
+ /// active in the current context.
+ /// This includes declarations from parent contexts that have
+ /// not been overridden.
+ /// <para>
+ /// <strong>Note:</strong> if there is a default prefix, it will not be
+ /// returned in this enumeration; check for the default prefix
+ /// using the <see cref="GetUri" /> with an argument of "".
+ /// </para>
+ /// </summary>
+ /// <returns>An enumeration of prefixes (never empty).</returns>
+ /// <seealso cref="GetDeclaredPrefixes" />
+ /// <seealso cref="GetUri" />
+ public IEnumerable GetPrefixes()
+ {
+ return currentContext.GetPrefixes();
+ }
+
+ /// <summary>
+ /// Return one of the prefixes mapped to a Namespace URI.
+ /// <para>
+ /// If more than one prefix is currently mapped to the same
+ /// URI, this method will make an arbitrary selection; if you
+ /// want all of the prefixes, use the <see cref="GetPrefixes()" />
+ /// method instead.
+ /// </para>
+ /// <para>
+ /// <strong>Note:</strong> this will never return the empty (default) prefix;
+ /// to check for a default prefix, use the <see cref="GetUri" />
+ /// method with an argument of "".
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// the namespace URI
+ /// </param>
+ /// <returns>
+ /// one of the prefixes currently mapped to the URI supplied,
+ /// or null if none is mapped or if the URI is assigned to
+ /// the default namespace
+ /// </returns>
+ /// <seealso cref="GetPrefixes(string)" />
+ /// <seealso cref="GetUri" />
+ public string GetPrefix(string uri)
+ {
+ return currentContext.GetPrefix(uri);
+ }
+
+ /// <summary>
+ /// Return an enumeration of all prefixes for a given URI whose
+ /// declarations are active in the current context.
+ /// This includes declarations from parent contexts that have
+ /// not been overridden.
+ /// <para>
+ /// This method returns prefixes mapped to a specific Namespace
+ /// URI. The xml: prefix will be included. If you want only one
+ /// prefix that's mapped to the Namespace URI, and you don't care
+ /// which one you get, use the <see cref="GetPrefix" />
+ /// method instead.
+ /// </para>
+ /// <para>
+ /// <strong>Note:</strong> the empty (default) prefix is <em>never</em> included
+ /// in this enumeration; to check for the presence of a default
+ /// Namespace, use the <see cref="GetUri" /> method with an
+ /// argument of "".
+ /// </para>
+ /// </summary>
+ /// <param name="uri">
+ /// The Namespace URI.
+ /// </param>
+ /// <returns>An enumeration of prefixes (never empty).</returns>
+ /// <seealso cref="GetPrefix" />
+ /// <seealso cref="GetDeclaredPrefixes" />
+ /// <seealso cref="GetUri" />
+ public IEnumerable GetPrefixes(string uri)
+ {
+ var prefixes = new ArrayList();
+ IEnumerator allPrefixes = GetPrefixes().GetEnumerator();
+ while (allPrefixes.MoveNext())
+ {
+ var prefix = (string)allPrefixes.Current;
+ if (uri.Equals(GetUri(prefix)))
+ {
+ prefixes.Add(prefix);
+ }
+ }
+ return prefixes;
+ }
+
+ /// <summary>
+ /// Return an enumeration of all prefixes declared in this context.
+ /// <para>
+ /// The empty (default) prefix will be included in this
+ /// enumeration; note that this behaviour differs from that of
+ /// <see cref="GetPrefix" /> and <see cref="GetPrefixes()" />.
+ /// </para>
+ /// </summary>
+ /// <returns>
+ /// An enumeration of all prefixes declared in this
+ /// context.
+ /// </returns>
+ /// <seealso cref="GetPrefixes()" />
+ /// <seealso cref="GetUri" />
+ public IEnumerable GetDeclaredPrefixes()
+ {
+ return currentContext.GetDeclaredPrefixes();
+ }
+
+ /// <summary>
+ /// Controls whether namespace declaration attributes are placed
+ /// into the <see cref="NSDECL" /> namespace
+ /// by <see cref="ProcessName" />. This may only be
+ /// changed before any contexts have been pushed.
+ /// </summary>
+ /// <exception cref="InvalidOperationException">
+ /// when attempting to set this
+ /// after any context has been pushed.
+ /// </exception>
+ public void SetNamespaceDeclUris(bool value)
+ {
+ if (contextPos != 0)
+ {
+ throw new InvalidOperationException();
+ }
+ if (value == namespaceDeclUris)
+ {
+ return;
+ }
+ namespaceDeclUris = value;
+ if (value)
+ {
+ currentContext.DeclarePrefix("xmlns", NSDECL);
+ }
+ else
+ {
+ contexts[contextPos] = currentContext = new Context(this);
+ currentContext.DeclarePrefix("xml", XMLNS);
+ }
+ }
+
+ /// <summary>
+ /// Returns true if namespace declaration attributes are placed into
+ /// a namespace. This behavior is not the default.
+ /// </summary>
+ public bool IsNamespaceDeclUris
+ {
+ get { return namespaceDeclUris; }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////////
+
+ private Context[] contexts;
+ private Context currentContext;
+ private int contextPos;
+ private bool namespaceDeclUris;
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal classes.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Internal class for a single Namespace context.
+ /// </summary>
+ /// <remarks>
+ /// This module caches and reuses Namespace contexts,
+ /// so the number allocated
+ /// will be equal to the element depth of the document, not to the total
+ /// number of elements (i.e. 5-10 rather than tens of thousands).
+ /// Also, data structures used to represent contexts are shared when
+ /// possible (child contexts without declarations) to further reduce
+ /// the amount of memory that's consumed.
+ /// </remarks>
+ internal sealed class Context
+ {
+ private readonly NamespaceSupport outerInstance;
+
+ /// <summary>
+ /// Create the root-level Namespace context.
+ /// </summary>
+ public Context(NamespaceSupport outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ CopyTables();
+ }
+
+ /// <summary>
+ /// (Re)set the parent of this Namespace context.
+ /// The context must either have been freshly constructed,
+ /// or must have been cleared.
+ /// </summary>
+ /// <param name="parent">The parent Namespace context object.</param>
+ public void SetParent(Context parent)
+ {
+ this.parent = parent;
+ declarations = null;
+ prefixTable = parent.prefixTable;
+ uriTable = parent.uriTable;
+ elementNameTable = parent.elementNameTable;
+ attributeNameTable = parent.attributeNameTable;
+ defaultNs = parent.defaultNs;
+ declSeen = false;
+ declsOK = true;
+ }
+
+ /// <summary>
+ /// Makes associated state become collectible,
+ /// invalidating this context.
+ /// <see cref="SetParent(Context)"/> must be called before
+ /// this context may be used again.
+ /// </summary>
+ public void Clear()
+ {
+ parent = null;
+ prefixTable = null;
+ uriTable = null;
+ elementNameTable = null;
+ attributeNameTable = null;
+ defaultNs = null;
+ }
+
+ /// <summary>
+ /// Declare a Namespace prefix for this context.
+ /// </summary>
+ /// <param name="prefix">The prefix to declare.</param>
+ /// <param name="uri">The associated Namespace URI.</param>
+ /// <seealso cref="DeclarePrefix(string, string)"/>
+ public void DeclarePrefix(string prefix, string uri)
+ {
+ // Lazy processing...
+ if (!declsOK)
+ {
+ throw new InvalidOperationException("can't declare any more prefixes in this context");
+ }
+ if (!declSeen)
+ {
+ CopyTables();
+ }
+ if (declarations == null)
+ {
+ declarations = new List<string>();
+ }
+
+ prefix = prefix.Intern();
+ uri = uri.Intern();
+ if ("".Equals(prefix))
+ {
+ if ("".Equals(uri))
+ {
+ defaultNs = null;
+ }
+ else
+ {
+ defaultNs = uri;
+ }
+ }
+ else
+ {
+ prefixTable.Add(prefix, uri);
+ uriTable.Add(uri, prefix); // may wipe out another prefix
+ }
+ declarations.Add(prefix);
+ }
+
+ /// <summary>
+ /// Process an XML qualified name in this context.
+ /// </summary>
+ /// <param name="qName">The XML qualified name.</param>
+ /// <param name="isAttribute">true if this is an attribute name.</param>
+ /// <param name="namespaceDeclUris"></param>
+ /// <returns>An array of three strings containing the
+ /// URI part (or empty string), the local part,
+ /// and the raw name, all internalized, or null
+ /// if there is an undeclared prefix.</returns>
+ /// <seealso cref="NamespaceSupport.ProcessName(string, bool)"/>
+ internal string[] ProcessName(string qName, bool isAttribute)
+ {
+ string[] name;
+ IDictionary<string, string[]> table;
+
+ // detect errors in call sequence
+ declsOK = false;
+
+ // Select the appropriate table.
+ if (isAttribute)
+ {
+ table = attributeNameTable;
+ }
+ else
+ {
+ table = elementNameTable;
+ }
+
+ // Start by looking in the cache, and
+ // return immediately if the name
+ // is already known in this content
+ if (table.ContainsKey(qName))
+ {
+ return (string[])table[qName];
+ }
+
+ // We haven't seen this name in this
+ // context before. Maybe in the parent
+ // context, but we can't assume prefix
+ // bindings are the same.
+ name = new string[3];
+ name[2] = qName.Intern();
+ int index = qName.IndexOf(':');
+
+ // No prefix.
+ if (index == -1)
+ {
+ if (isAttribute)
+ {
+ if (qName == "xmlns" && outerInstance.namespaceDeclUris)
+ {
+ name[0] = NSDECL;
+ }
+ else
+ {
+ name[0] = "";
+ }
+ }
+ else if (defaultNs == null)
+ {
+ name[0] = "";
+ }
+ else
+ {
+ name[0] = defaultNs;
+ }
+ name[1] = name[2];
+ }
+
+ // Prefix
+ else
+ {
+ string prefix = qName.Substring(0, index);
+ string local = qName.Substring(index + 1);
+ string uri = null;
+ if ("".Equals(prefix))
+ {
+ uri = defaultNs;
+ }
+ else if (prefixTable.ContainsKey(prefix))
+ {
+ uri = (string)prefixTable[prefix];
+ }
+ if (uri == null || (!isAttribute && "xmlns".Equals(prefix)))
+ {
+ return null;
+ }
+ name[0] = uri;
+ name[1] = local.Intern();
+ }
+
+ // Save in the cache for future use.
+ // (Could be shared with parent context...)
+ table.Add(name[2], name);
+ return name;
+ }
+
+ /// <summary>
+ /// Look up the URI associated with a prefix in this context.
+ /// </summary>
+ /// <param name="prefix">The prefix to look up.</param>
+ /// <returns>The associated Namespace URI, or null if none is declared.</returns>
+ /// <seealso cref="NamespaceSupport.GetUri(string)"/>
+ internal string GetURI(string prefix)
+ {
+ if ("".Equals(prefix))
+ {
+ return defaultNs;
+ }
+ if (prefixTable == null)
+ {
+ return null;
+ }
+ if (prefixTable.ContainsKey(prefix))
+ {
+ return (string)prefixTable[prefix];
+ }
+ return null;
+ }
+
+ /// <summary>
+ /// Look up one of the prefixes associated with a URI in this context.
+ /// <para/>
+ /// Since many prefixes may be mapped to the same URI, the return value may be unreliable.
+ /// </summary>
+ /// <param name="uri">The URI to look up.</param>
+ /// <returns>The associated prefix, or null if none is declared.</returns>
+ /// <seealso cref="NamespaceSupport.GetPrefix(string)"/>
+ internal string GetPrefix(string uri)
+ {
+ if (uriTable == null)
+ {
+ return null;
+ }
+ if (uriTable.ContainsKey(uri))
+ {
+ return (string)uriTable[uri];
+ }
+ return null;
+ }
+
+ /// <summary>
+ /// Return an enumeration of prefixes declared in this context.
+ /// </summary>
+ /// <returns>An enumeration of prefixes (possibly empty).</returns>
+ /// <seealso cref="NamespaceSupport.GetDeclaredPrefixes()"/>
+ internal IEnumerable GetDeclaredPrefixes()
+ {
+ if (declarations == null)
+ {
+ return Enumerable.Empty<object>();
+ }
+ return declarations;
+ }
+
+ /// <summary>
+ /// Return an enumeration of all prefixes currently in force.
+ /// <para/>
+ /// The default prefix, if in force, is <em>not</em>
+ /// returned, and will have to be checked for separately.
+ /// </summary>
+ /// <returns>An enumeration of prefixes (never empty).</returns>
+ /// <seealso cref="NamespaceSupport.GetPrefixes()"/>
+ internal IEnumerable GetPrefixes()
+ {
+ if (prefixTable == null)
+ {
+ return Enumerable.Empty<object>();
+ }
+ return prefixTable.Keys;
+ }
+
+ ////////////////////////////////////////////////////////////////
+ // Internal methods.
+ ////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Copy on write for the internal tables in this context.
+ /// <para/>
+ /// This class is optimized for the normal case where most
+ /// elements do not contain Namespace declarations.
+ /// </summary>
+ private void CopyTables()
+ {
+ if (prefixTable != null)
+ {
+ prefixTable = (Hashtable)prefixTable.Clone();
+ }
+ else
+ {
+ prefixTable = new Hashtable();
+ }
+ if (uriTable != null)
+ {
+ uriTable = (Hashtable)uriTable.Clone();
+ }
+ else
+ {
+ uriTable = new Hashtable();
+ }
+ elementNameTable = new Dictionary<string, string[]>();
+ attributeNameTable = new Dictionary<string, string[]>();
+ declSeen = true;
+ }
+
+ ////////////////////////////////////////////////////////////////
+ // Protected state.
+ ////////////////////////////////////////////////////////////////
+
+ private Hashtable prefixTable;
+ private Hashtable uriTable;
+ private IDictionary<string, string[]> elementNameTable;
+ private IDictionary<string, string[]> attributeNameTable;
+ private string defaultNs;
+ internal bool declsOK = true;
+
+ ////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////
+
+ private IList<string> declarations;
+ private bool declSeen;
+ private Context parent;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Helpers/XMLFilterImpl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Helpers/XMLFilterImpl.cs b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/XMLFilterImpl.cs
new file mode 100644
index 0000000..cbf9e6d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/XMLFilterImpl.cs
@@ -0,0 +1,587 @@
+// XMLFilterImpl.java - base SAX2 filter implementation.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the Public Domain.
+// $Id: XMLFilterImpl.java,v 1.9 2004/04/26 17:34:35 dmegginson Exp $
+
+using System;
+
+namespace Sax.Helpers
+{
+ /// <summary>
+ /// Base class for deriving an XML filter.
+ /// <para/>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// See <a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class is designed to sit between an <see cref="IXMLReader" />
+ /// and the client application's event handlers. By default, it
+ /// does nothing but pass requests up to the reader and events
+ /// on to the handlers unmodified, but subclasses can override
+ /// specific methods to modify the event stream or the configuration
+ /// requests as they pass through.
+ /// </summary>
+ /// <seealso cref="IXMLFilter" />
+ /// <seealso cref="IXMLReader" />
+ /// <seealso cref="IEntityResolver" />
+ /// <seealso cref="IDTDHandler" />
+ /// <seealso cref="IContentHandler" />
+ /// <seealso cref="IErrorHandler" />
+ public class XMLFilter : IXMLFilter, IEntityResolver, IDTDHandler, IContentHandler, IErrorHandler
+ {
+ /// <summary>
+ /// Construct an empty XML filter, with no parent.
+ /// <para>
+ /// This filter will have no parent: you must assign a parent
+ /// before you start a parse or do any configuration with
+ /// setFeature or setProperty, unless you use this as a pure event
+ /// consumer rather than as an <see cref="IXmlReader" />.
+ /// </para>
+ /// </summary>
+ /// <seealso cref="IXMLReader.SetFeature" />
+ /// <seealso cref="IXMLReader.SetProperty" />
+ /// <seealso cref="Parent" />
+ public XMLFilter()
+ {
+ }
+
+ /// <summary>
+ /// Construct an XML filter with the specified parent.
+ /// </summary>
+ /// <param name="parent">The parent</param>
+ /// <seealso cref="Parent" />
+ public XMLFilter(IXMLReader parent)
+ {
+ this.parent = parent;
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of IXMLFilter.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Gets or sets the parent reader.
+ /// </summary>
+ /// <remarks>
+ /// This is the <see cref="IXMLReader"/> from which
+ /// this filter will obtain its events and to which it will pass its
+ /// configuration requests. The parent may itself be another filter.
+ /// <para/>
+ /// If there is no parent reader set, any attempt to parse
+ /// or to set or get a feature or property will fail.
+ /// </remarks>
+ public IXMLReader Parent
+ {
+ get { return parent; }
+ set { parent = value; }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of IXMLReader.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Set the value of a feature.
+ /// <para/>
+ /// This will always fail if the parent is null.
+ /// </summary>
+ /// <param name="name">The feature name.</param>
+ /// <param name="value">The requested feature value.</param>
+ /// <exception cref="SAXNotRecognizedException">If the feature
+ /// value can't be assigned or retrieved from the parent.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// parent recognizes the feature name but
+ /// cannot set the requested value.</exception>
+ public virtual void SetFeature(string name, bool value)
+ {
+ if (parent != null)
+ {
+ parent.SetFeature(name, value);
+ }
+ else
+ {
+ throw new SAXNotRecognizedException("Feature: " + name);
+ }
+ }
+
+ /// <summary>
+ /// Look up the value of a feature.
+ /// <para/>
+ /// This will always fail if the parent is null.
+ /// </summary>
+ /// <param name="name">The feature name.</param>
+ /// <returns>The current value of the feature.</returns>
+ /// <exception cref="SAXNotRecognizedException">If the feature
+ /// value can't be assigned or retrieved from the parent.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// parent recognizes the feature name but
+ /// cannot determine its value at this time.</exception>
+ public virtual bool GetFeature(string name)
+ {
+ if (parent != null)
+ {
+ return parent.GetFeature(name);
+ }
+ throw new SAXNotRecognizedException("Feature: " + name);
+ }
+
+ /// <summary>
+ /// Set the value of a property.
+ /// <para/>
+ /// This will always fail if the parent is null.
+ /// </summary>
+ /// <param name="name">The property name.</param>
+ /// <param name="value">The requested property value.</param>
+ /// <exception cref="SAXNotRecognizedException">If the property
+ /// value can't be assigned or retrieved from the parent.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// parent recognizes the property name but
+ /// cannot set the requested value.</exception>
+ public virtual void SetProperty(string name, object value)
+ {
+ if (parent != null)
+ {
+ parent.SetProperty(name, value);
+ }
+ else
+ {
+ throw new SAXNotRecognizedException("Property: " + name);
+ }
+ }
+
+ /// <summary>
+ /// Look up the value of a property.
+ /// </summary>
+ /// <param name="name">The property name.</param>
+ /// <returns>The current value of the property.</returns>
+ /// <exception cref="SAXNotRecognizedException">If the property
+ /// value can't be assigned or retrieved from the parent.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// parent recognizes the property name but
+ /// cannot determine its value at this time.</exception>
+ public virtual object GetProperty(string name)
+ {
+ if (parent != null)
+ {
+ return parent.GetProperty(name);
+ }
+ throw new SAXNotRecognizedException("Property: " + name);
+ }
+
+ /// <summary>
+ /// Gets or Sets the entity resolver.
+ /// </summary>
+ public IEntityResolver EntityResolver
+ {
+ get { return entityResolver; }
+ set { entityResolver = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the DTD event handler.
+ /// </summary>
+ public IDTDHandler DTDHandler
+ {
+ get { return dtdHandler; }
+ set { dtdHandler = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the content event handler.
+ /// </summary>
+ public IContentHandler ContentHandler
+ {
+ get { return contentHandler; }
+ set { contentHandler = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the error event handler.
+ /// </summary>
+ public IErrorHandler ErrorHandler
+ {
+ get { return errorHandler; }
+ set { errorHandler = value; }
+ }
+
+ /// <summary>
+ /// Parse an XML document.
+ /// <para>
+ /// The application can use this method to instruct the XML
+ /// reader to begin parsing an XML document from any valid input
+ /// source (a character stream, a byte stream, or a URI).
+ /// </para>
+ /// <para>
+ /// Applications may not invoke this method while a parse is in
+ /// progress (they should create a new XMLReader instead for each
+ /// nested XML document). Once a parse is complete, an
+ /// application may reuse the same XMLReader object, possibly with a
+ /// different input source.
+ /// Configuration of the XMLReader object (such as handler bindings and
+ /// values established for feature flags and properties) is unchanged
+ /// by completion of a parse, unless the definition of that aspect of
+ /// the configuration explicitly specifies other behavior.
+ /// (For example, feature flags or properties exposing
+ /// characteristics of the document being parsed.)
+ /// </para>
+ /// <para>
+ /// During the parse, the XMLReader will provide information
+ /// about the XML document through the registered event
+ /// handlers.
+ /// </para>
+ /// <para>
+ /// This method is synchronous: it will not return until parsing
+ /// has ended. If a client application wants to terminate
+ /// parsing early, it should throw an exception.
+ /// </para>
+ /// </summary>
+ /// <param name="input">
+ /// The input source for the top-level of the
+ /// XML document.
+ /// </param>
+ /// <exception cref="SAXException">
+ /// Any SAX exception, possibly
+ /// wrapping another exception.
+ /// </exception>
+ /// <exception cref="System.IO.IOException">
+ /// An IO exception from the parser,
+ /// possibly from a byte stream or character stream
+ /// supplied by the application.
+ /// </exception>
+ /// <seealso cref="InputSource" />
+ /// <seealso cref="Parse(string)" />
+ /// <seealso cref="IEntityResolver" />
+ /// <seealso cref="IDTDHandler" />
+ /// <seealso cref="IContentHandler" />
+ /// <seealso cref="IErrorHandler" />
+ public virtual void Parse(InputSource input)
+ {
+ SetupParse();
+ parent.Parse(input);
+ }
+
+ /// <summary>
+ /// Parse a document.
+ /// </summary>
+ /// <param name="systemId">The system identifier as a fully-qualified URI.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <exception cref="System.IO.IOException">An IO exception from the parser,
+ /// possibly from a byte stream or character stream
+ /// supplied by the application.</exception>
+ public virtual void Parse(string systemId)
+ {
+ Parse(new InputSource(systemId));
+ }
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of IEntityResolver.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Filter an external entity resolution.
+ /// </summary>
+ /// <param name="publicId">The entity's public identifier, or null.</param>
+ /// <param name="systemId">The entity's system identifier.</param>
+ /// <returns>A new <see cref="InputSource"/> or null for the default.</returns>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ /// <exception cref="System.IO.IOException">The client may throw an
+ /// I/O-related exception while obtaining the
+ /// new <see cref="InputSource"/>.</exception>
+ public virtual InputSource ResolveEntity(string publicId, string systemId)
+ {
+ if (entityResolver != null)
+ {
+ return entityResolver.ResolveEntity(publicId, systemId);
+ }
+ return null;
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of IDTDHandler.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Filter a notation declaration event.
+ /// </summary>
+ /// <param name="name">The notation name.</param>
+ /// <param name="publicId">The notation's public identifier, or null.</param>
+ /// <param name="systemId">The notation's system identifier, or null.</param>
+ /// <seealso cref="SAXException">The client may throw
+ /// an exception during processing.</seealso>
+ public virtual void NotationDecl(string name, string publicId, string systemId)
+ {
+ if (dtdHandler != null)
+ {
+ dtdHandler.NotationDecl(name, publicId, systemId);
+ }
+ }
+
+ /// <summary>
+ /// Filter an unparsed entity declaration event.
+ /// </summary>
+ /// <param name="name">The entity name.</param>
+ /// <param name="publicId">The entity's public identifier, or null.</param>
+ /// <param name="systemId">The entity's system identifier, or null.</param>
+ /// <param name="notationName">The name of the associated notation.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void UnparsedEntityDecl(string name, string publicId, string systemId, string notationName)
+ {
+ if (dtdHandler != null)
+ {
+ dtdHandler.UnparsedEntityDecl(name, publicId, systemId, notationName);
+ }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of IContentHandler.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Filter a new document locator event.
+ /// </summary>
+ /// <param name="locator">The document locator.</param>
+ public virtual void SetDocumentLocator(ILocator locator)
+ {
+ this.locator = locator;
+ if (contentHandler != null)
+ {
+ contentHandler.SetDocumentLocator(locator);
+ }
+ }
+
+ /// <summary>
+ /// Filter a start document event.
+ /// </summary>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void StartDocument()
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.StartDocument();
+ }
+ }
+
+ /// <summary>
+ /// Filter an end document event.
+ /// </summary>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void EndDocument()
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.EndDocument();
+ }
+ }
+
+ /// <summary>
+ /// Filter a start Namespace prefix mapping event.
+ /// </summary>
+ /// <param name="prefix">The Namespace prefix.</param>
+ /// <param name="uri">The Namespace URI.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void StartPrefixMapping(string prefix, string uri)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.StartPrefixMapping(prefix, uri);
+ }
+ }
+
+ /// <summary>
+ /// Filter an end Namespace prefix mapping event.
+ /// </summary>
+ /// <param name="prefix">The Namespace prefix.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void EndPrefixMapping(string prefix)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.EndPrefixMapping(prefix);
+ }
+ }
+
+ /// <summary>
+ /// Filter a start element event.
+ /// </summary>
+ /// <param name="uri">The element's Namespace URI, or the empty string.</param>
+ /// <param name="localName">The element's local name, or the empty string.</param>
+ /// <param name="qName">The element's qualified (prefixed) name, or the empty string.</param>
+ /// <param name="atts">The element's attributes.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void StartElement(string uri, string localName, string qName, IAttributes atts)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.StartElement(uri, localName, qName, atts);
+ }
+ }
+
+ /// <summary>
+ /// Filter an end element event.
+ /// </summary>
+ /// <param name="uri">The element's Namespace URI, or the empty string.</param>
+ /// <param name="localName">The element's local name, or the empty string.</param>
+ /// <param name="qName">The element's qualified (prefixed) name, or the empty string.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void EndElement(string uri, string localName, string qName)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.EndElement(uri, localName, qName);
+ }
+ }
+
+ /// <summary>
+ /// Filter a character data event.
+ /// </summary>
+ /// <param name="ch">An array of characters.</param>
+ /// <param name="start">The starting position in the array.</param>
+ /// <param name="length">The number of characters to use from the array.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void Characters(char[] ch, int start, int length)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.Characters(ch, start, length);
+ }
+ }
+
+ /// <summary>
+ /// Filter an ignorable whitespace event.
+ /// </summary>
+ /// <param name="ch">An array of characters.</param>
+ /// <param name="start">The starting position in the array.</param>
+ /// <param name="length">The number of characters to use from the array.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void IgnorableWhitespace(char[] ch, int start, int length)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.IgnorableWhitespace(ch, start, length);
+ }
+ }
+
+ /// <summary>
+ /// Filter a processing instruction event.
+ /// </summary>
+ /// <param name="target">The processing instruction target.</param>
+ /// <param name="data">The text following the target.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void ProcessingInstruction(string target, string data)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.ProcessingInstruction(target, data);
+ }
+ }
+
+ /// <summary>
+ /// Filter a skipped entity event.
+ /// </summary>
+ /// <param name="name">The name of the skipped entity.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void SkippedEntity(string name)
+ {
+ if (contentHandler != null)
+ {
+ contentHandler.SkippedEntity(name);
+ }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of IErrorHandler.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Filter a warning event.
+ /// </summary>
+ /// <param name="e">The warning as an exception.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void Warning(SAXParseException e)
+ {
+ if (errorHandler != null)
+ {
+ errorHandler.Warning(e);
+ }
+ }
+
+ /// <summary>
+ /// Filter an error event.
+ /// </summary>
+ /// <param name="e">The error as an exception.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void Error(SAXParseException e)
+ {
+ if (errorHandler != null)
+ {
+ errorHandler.Error(e);
+ }
+ }
+
+ /// <summary>
+ /// Filter a fatal error event.
+ /// </summary>
+ /// <param name="e">The error as an exception.</param>
+ /// <exception cref="SAXException">The client may throw
+ /// an exception during processing.</exception>
+ public virtual void FatalError(SAXParseException e)
+ {
+ if (errorHandler != null)
+ {
+ errorHandler.FatalError(e);
+ }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal methods.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Set up before a parse.
+ /// <para/>
+ /// Before every parse, check whether the parent is
+ /// non-null, and re-register the filter for all of the
+ /// events.
+ /// </summary>
+ private void SetupParse()
+ {
+ if (parent == null)
+ {
+ throw new InvalidOperationException("No parent for filter");
+ }
+ parent.EntityResolver = this;
+ parent.DTDHandler = this;
+ parent.ContentHandler = this;
+ parent.ErrorHandler = this;
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////////
+
+ private IXMLReader parent = null;
+ private ILocator locator = null;
+ private IEntityResolver entityResolver = null;
+ private IDTDHandler dtdHandler = null;
+ private IContentHandler contentHandler = null;
+ private IErrorHandler errorHandler = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/InputSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/InputSource.cs b/src/Lucene.Net.Benchmark/Support/Sax/InputSource.cs
new file mode 100644
index 0000000..00b3490
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/InputSource.cs
@@ -0,0 +1,242 @@
+// SAX input source.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: InputSource.java,v 1.10 2004/12/02 02:49:58 dmegginson Exp $
+
+using System.IO;
+using System.Text;
+
+namespace Sax
+{
+ /// <summary>
+ /// A single input source for an XML entity.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class allows a SAX application to encapsulate information
+ /// about an input source in a single object, which may include
+ /// a public identifier, a system identifier, a byte stream (possibly
+ /// with a specified encoding), and/or a character stream.
+ /// <para/>
+ /// There are two places that the application can deliver an
+ /// input source to the parser: as the argument to the <see cref="IParser.Parse(InputSource)"/>
+ /// method, or as the return value of the <see cref="IEntityResolver.ResolveEntity(string, string)"/>
+ /// method.
+ /// <para/>
+ /// The SAX parser will use the InputSource object to determine how
+ /// to read XML input. If there is a character stream available, the
+ /// parser will read that stream directly, disregarding any text
+ /// encoding declaration found in that stream.
+ /// If there is no character stream, but there is
+ /// a byte stream, the parser will use that byte stream, using the
+ /// encoding specified in the <see cref="InputSource"/> or else (if no encoding is
+ /// specified) autodetecting the character encoding using an algorithm
+ /// such as the one in the XML specification. If neither a character
+ /// stream nor a
+ /// byte stream is available, the parser will attempt to open a URL
+ /// connection to the resource identified by the system
+ /// identifier.
+ /// <para/>
+ /// An <see cref="InputSource"/> object belongs to the application: the SAX parser
+ /// shall never modify it in any way (it may modify a copy if
+ /// necessary). However, standard processing of both byte and
+ /// character streams is to close them on as part of end-of-parse cleanup,
+ /// so applications should not attempt to re-use such streams after they
+ /// have been handed to a parser.
+ /// </remarks>
+ /// <since>SAX 1.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="IXMLReader.Parse(InputSource)"/>
+ /// <seealso cref="IEntityResolver.ResolveEntity(string, string)"/>
+ /// <seealso cref="System.IO.Stream"/>
+ /// <seealso cref="System.IO.TextReader"/>
+ public class InputSource
+ {
+ /// <summary>
+ /// Zero-argument default constructor.
+ /// </summary>
+ /// <seealso cref="PublicId"/>
+ /// <seealso cref="SystemId"/>
+ /// <seealso cref="Stream"/>
+ /// <seealso cref="TextReader"/>
+ /// <seealso cref="Encoding"/>
+ public InputSource()
+ {
+ }
+
+ /// <summary>
+ /// Create a new input source with a system identifier.
+ /// </summary>
+ /// <remarks>
+ /// Applications may use <see cref="PublicId"/> to include a
+ /// public identifier as well, or <see cref="Encoding"/> to specify
+ /// the character encoding, if known.
+ /// <para/>
+ /// If the system identifier is a URL, it must be fully
+ /// resolved (it may not be a relative URL).
+ /// </remarks>
+ /// <param name="systemId">The system identifier (URI).</param>
+ /// <seealso cref="PublicId"/>
+ /// <seealso cref="SystemId"/>
+ /// <seealso cref="Stream"/>
+ /// <seealso cref="TextReader"/>
+ /// <seealso cref="Encoding"/>
+ public InputSource(string systemId)
+ {
+ this.systemId = systemId;
+ }
+
+ /// <summary>
+ /// Create a new input source with a byte stream.
+ /// </summary>
+ /// <remarks>
+ /// Application writers should use <see cref="SystemId"/> to provide a base
+ /// for resolving relative URIs, may use <see cref="PublicId"/> to include a
+ /// public identifier, and may use <see cref="Encoding"/> to specify the object's
+ /// character encoding.
+ /// </remarks>
+ /// <param name="byteStream">The raw byte stream containing the document.</param>
+ /// <seealso cref="PublicId"/>
+ /// <seealso cref="SystemId"/>
+ /// <seealso cref="Encoding"/>
+ /// <seealso cref="Stream"/>
+ /// <seealso cref="TextReader"/>
+ public InputSource(Stream byteStream)
+ {
+ this.byteStream = byteStream;
+ }
+
+ /// <summary>
+ /// Create a new input source with a character stream.
+ /// </summary>
+ /// <remarks>
+ /// Application writers should use <see cref="SystemId"/> to provide a base
+ /// for resolving relative URIs, and may use <see cref="PublicId"/> to include a
+ /// public identifier.
+ /// <para/>
+ /// The character stream shall not include a byte order mark.
+ /// </remarks>
+ /// <param name="characterStream"></param>
+ /// <seealso cref="PublicId"/>
+ /// <seealso cref="SystemId"/>
+ /// <seealso cref="Stream"/>
+ /// <seealso cref="TextReader"/>
+ public InputSource(TextReader characterStream)
+ {
+ this.characterStream = characterStream;
+ }
+
+ /// <summary>
+ /// Gets or Sets the public identifier for this input source.
+ /// </summary>
+ /// <remarks>
+ /// The public identifier is always optional: if the application
+ /// writer includes one, it will be provided as part of the
+ /// location information.
+ /// </remarks>
+ /// <seealso cref="ILocator.PublicId"/>
+ /// <seealso cref="SAXParseException.PublicId"/>
+ public virtual string PublicId
+ {
+ get { return publicId; }
+ set { publicId = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the system identifier for this input source.
+ /// </summary>
+ /// <remarks>
+ /// The system identifier is optional if there is a byte stream
+ /// or a character stream, but it is still useful to provide one,
+ /// since the application can use it to resolve relative URIs
+ /// and can include it in error messages and warnings(the parser
+ /// will attempt to open a connection to the URI only if
+ /// there is no byte stream or character stream specified).
+ /// <para/>
+ /// If the application knows the character encoding of the
+ /// object pointed to by the system identifier, it can register
+ /// the encoding using the <see cref="Encoding"/> property setter.
+ /// <para/>
+ /// If the system identifier is a URL, it must be fully
+ /// resolved(it may not be a relative URL).
+ /// </remarks>
+ /// <seealso cref="Encoding"/>
+ /// <seealso cref="SystemId"/>
+ /// <seealso cref="ILocator.SystemId"/>
+ /// <seealso cref="SAXParseException.SystemId"/>
+ public virtual string SystemId
+ {
+ get { return systemId; }
+ set { systemId = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the byte stream for this input source.
+ /// </summary>
+ /// <remarks>
+ /// The SAX parser will ignore this if there is also a character
+ /// stream specified, but it will use a byte stream in preference
+ /// to opening a URI connection itself.
+ /// <para/>
+ /// If the application knows the character encoding of the
+ /// byte stream, it should set it with the setEncoding method.
+ /// </remarks>
+ /// <seealso cref="Encoding"/>
+ /// <seealso cref="Stream"/>
+ /// <seealso cref="System.IO.Stream"/>
+ public virtual Stream Stream
+ {
+ get { return byteStream; }
+ set { byteStream = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the character encoding.
+ /// </summary>
+ /// <remarks>
+ /// The encoding must be a string acceptable for an
+ /// XML encoding declaration(see section 4.3.3 of the XML 1.0
+ /// recommendation).
+ /// <para/>
+ /// This method has no effect when the application provides a
+ /// character stream.
+ /// </remarks>
+ /// <seealso cref="SystemId"/>
+ /// <seealso cref="Stream"/>
+ public virtual Encoding Encoding
+ {
+ get { return encoding; }
+ set { encoding = value; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the character stream for this input source.
+ /// </summary>
+ /// <remarks>
+ /// If there is a character stream specified, the SAX parser
+ /// will ignore any byte stream and will not attempt to open
+ /// a URI connection to the system identifier.
+ /// </remarks>
+ /// <seealso cref="System.IO.TextReader"/>
+ public virtual TextReader TextReader
+ {
+ get { return characterStream; }
+ set { characterStream = value; }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////////
+
+ private string publicId;
+ private string systemId;
+ private Stream byteStream;
+ private Encoding encoding;
+ private TextReader characterStream;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Locator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Locator.cs b/src/Lucene.Net.Benchmark/Support/Sax/Locator.cs
new file mode 100644
index 0000000..25d1fce
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Locator.cs
@@ -0,0 +1,125 @@
+// SAX locator interface for document events.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: Locator.java,v 1.8 2002/01/30 21:13:47 dbrownell Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Interface for associating a SAX event with a document location.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// See <a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// If a SAX parser provides location information to the SAX
+ /// application, it does so by implementing this interface and then
+ /// passing an instance to the application using the content
+ /// handler's <see cref="IContentHandler.SetDocumentLocator(ILocator)"/>
+ /// method.The application can use the
+ /// object to obtain the location of any other SAX event
+ /// in the XML source document.
+ /// <para/>
+ /// Note that the results returned by the object will be valid only
+ /// during the scope of each callback method: the application
+ /// will receive unpredictable results if it attempts to use the
+ /// locator at any other time, or after parsing completes.
+ /// <para/>
+ /// SAX parsers are not required to supply a locator, but they are
+ /// very strongly encouraged to do so. If the parser supplies a
+ /// locator, it must do so before reporting any other document events.
+ /// If no locator has been set by the time the application receives
+ /// the <see cref="IContentHandler.StartDocument()"/>
+ /// event, the application should assume that a locator is not
+ /// available.
+ /// </remarks>
+ /// <since>SAX 1.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="IContentHandler.SetDocumentLocator(ILocator)"/>
+ public interface ILocator
+ {
+ /// <summary>
+ /// Gets the public identifier for the current document event.
+ /// <para/>
+ /// The return value is the public identifier of the document
+ /// entity or of the external parsed entity in which the markup
+ /// triggering the event appears.
+ /// <para/>
+ /// Returns a string containing the public identifier, or null if none is available.
+ /// </summary>
+ /// <seealso cref="SystemId"/>
+ string PublicId { get; }
+
+ /// <summary>
+ /// Return the system identifier for the current document event.
+ /// <para/>
+ /// The return value is the system identifier of the document
+ /// entity or of the external parsed entity in which the markup
+ /// triggering the event appears.
+ /// <para/>If the system identifier is a URL, the parser must resolve it
+ /// fully before passing it to the application. For example, a file
+ /// name must always be provided as a <em>file:...</em> URL, and other
+ /// kinds of relative URI are also resolved against their bases.
+ /// <para/>
+ /// Returns a string containing the system identifier, or null
+ /// if none is available.
+ /// </summary>
+ /// <seealso cref="PublicId"/>
+ string SystemId { get; }
+
+ /// <summary>
+ /// Return the line number where the current document event ends.
+ /// Lines are delimited by line ends, which are defined in
+ /// the XML specification.
+ /// <para/>
+ /// <strong>Warning:</strong> The return value from the method
+ /// is intended only as an approximation for the sake of diagnostics;
+ /// it is not intended to provide sufficient information
+ /// to edit the character content of the original XML document.
+ /// In some cases, these "line" numbers match what would be displayed
+ /// as columns, and in others they may not match the source text
+ /// due to internal entity expansion.
+ /// <para/>
+ /// The return value is an approximation of the line number
+ /// in the document entity or external parsed entity where the
+ /// markup triggering the event appears.
+ /// <para/>
+ /// If possible, the SAX driver should provide the line position
+ /// of the first character after the text associated with the document
+ /// event. The first line is line 1.
+ /// <para/>
+ /// Returns the line number, or -1 if none is available.
+ /// </summary>
+ /// <seealso cref="ColumnNumber"/>
+ int LineNumber { get; }
+
+ /// <summary>
+ /// Return the column number where the current document event ends.
+ /// This is one-based number of Java<code>char</code> values since
+ /// the last line end.
+ /// <para/>
+ /// <strong>Warning:</strong> The return value from the method
+ /// is intended only as an approximation for the sake of diagnostics;
+ /// it is not intended to provide sufficient information
+ /// to edit the character content of the original XML document.
+ /// For example, when lines contain combining character sequences, wide
+ /// characters, surrogate pairs, or bi-directional text, the value may
+ /// not correspond to the column in a text editor's display.
+ /// <para/>
+ /// The return value is an approximation of the column number
+ /// in the document entity or external parsed entity where the
+ /// markup triggering the event appears.
+ /// <para/>
+ /// If possible, the SAX driver should provide the line position
+ /// of the first character after the text associated with the document
+ /// event. The first column in each line is column 1.
+ /// <para/>
+ /// Returns the column number, or -1 if none is available.
+ /// </summary>
+ /// <seealso cref="LineNumber"/>
+ int ColumnNumber { get; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/SAXException.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/SAXException.cs b/src/Lucene.Net.Benchmark/Support/Sax/SAXException.cs
new file mode 100644
index 0000000..fc075d8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/SAXException.cs
@@ -0,0 +1,165 @@
+// SAX exception class.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: SAXException.java,v 1.7 2002/01/30 21:13:48 dbrownell Exp $
+
+using System;
+#if FEATURE_SERIALIZABLE
+using System.Runtime.Serialization;
+#endif
+
+namespace Sax
+{
+ /// <summary>
+ /// Encapsulate a general SAX error or warning.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class can contain basic error or warning information from
+ /// either the XML parser or the application: a parser writer or
+ /// application writer can subclass it to provide additional
+ /// functionality. SAX handlers may throw this exception or
+ /// any exception subclassed from it.
+ /// <para/>
+ /// If the application needs to pass through other types of
+ /// exceptions, it must wrap those exceptions in a <see cref="SAXException"/>
+ /// or an exception derived from a <see cref="SAXException"/>.
+ /// <para/>
+ /// If the parser or application needs to include information about a
+ /// specific location in an XML document, it should use the
+ /// <see cref="SAXParseException"/> subclass.
+ /// </remarks>
+ // LUCENENET: All exeption classes should be marked serializable
+#if FEATURE_SERIALIZABLE
+ [Serializable]
+#endif
+ public class SAXException : Exception
+ {
+ /// <summary>
+ /// Create a new <see cref="SAXException"/>.
+ /// </summary>
+ public SAXException()
+ : base()
+ {
+ this.exception = null;
+ }
+
+ /// <summary>
+ /// Create a new <see cref="SAXException"/>.
+ /// </summary>
+ /// <param name="message">The error or warning message.</param>
+ public SAXException(string message)
+ : base(message)
+ {
+ this.exception = null;
+ }
+
+ /// <summary>
+ /// Create a new <see cref="SAXException"/> wrapping an existing exception.
+ /// </summary>
+ /// <remarks>
+ /// The existing exception will be embedded in the new
+ /// one, and its message will become the default message for
+ /// the <see cref="SAXException"/>.
+ /// </remarks>
+ /// <param name="e">The exception to be wrapped in a <see cref="SAXException"/>.</param>
+ public SAXException(Exception e)
+ : base()
+ {
+ this.exception = e;
+ }
+
+ /// <summary>
+ /// Create a new <see cref="SAXException"/> from an existing exception.
+ /// </summary>
+ /// <remarks>
+ /// The existing exception will be embedded in the new
+ /// one, but the new exception will have its own message.
+ /// </remarks>
+ /// <param name="message">The detail message.</param>
+ /// <param name="e">The exception to be wrapped in a SAXException.</param>
+ public SAXException(string message, Exception e)
+ : base(message)
+ {
+ this.exception = e;
+ }
+
+#if FEATURE_SERIALIZABLE
+ /// <summary>
+ /// Initializes a new instance of this class with serialized data.
+ /// </summary>
+ /// <param name="info">The <see cref="SerializationInfo"/> that holds the serialized object data about the exception being thrown.</param>
+ /// <param name="context">The <see cref="StreamingContext"/> that contains contextual information about the source or destination.</param>
+ public SAXException(SerializationInfo info, StreamingContext context)
+ : base(info, context)
+ {
+ }
+#endif
+
+ /// <summary>
+ /// Return a detail message for this exception.
+ /// </summary>
+ /// <remarks>
+ /// If there is an embedded exception, and if the SAXException
+ /// has no detail message of its own, this method will return
+ /// the detail message from the embedded exception.
+ /// </remarks>
+ public override string Message
+ {
+ get
+ {
+ string message = base.Message;
+
+ if (message == null && exception != null)
+ {
+ return exception.Message;
+ }
+ else
+ {
+ return message;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Gets the embedded exception, if any, or <c>null</c> if there is none.
+ /// </summary>
+ public virtual Exception Exception
+ {
+ get { return exception; }
+ }
+
+ /// <summary>
+ /// Override ToString to pick up any embedded exception.
+ /// </summary>
+ /// <returns>A string representation of this exception.</returns>
+ public override string ToString()
+ {
+ if (exception != null)
+ {
+ return exception.ToString();
+ }
+ else
+ {
+ return base.ToString();
+ }
+ }
+
+
+
+ //////////////////////////////////////////////////////////////////////
+ // Internal state.
+ //////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// The embedded exception if tunnelling, or null.
+ /// </summary>
+ private Exception exception;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/SAXNotRecognizedException.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/SAXNotRecognizedException.cs b/src/Lucene.Net.Benchmark/Support/Sax/SAXNotRecognizedException.cs
new file mode 100644
index 0000000..e0ef4f5
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/SAXNotRecognizedException.cs
@@ -0,0 +1,66 @@
+// SAXNotRecognizedException.java - unrecognized feature or value.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the Public Domain.
+// $Id: SAXNotRecognizedException.java,v 1.7 2002/01/30 21:13:48 dbrownell Exp $
+
+using System;
+#if FEATURE_SERIALIZABLE
+using System.Runtime.Serialization;
+#endif
+
+namespace Sax
+{
+ /// <summary>
+ /// Exception class for an unrecognized identifier.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// An XMLReader will throw this exception when it finds an
+ /// unrecognized feature or property identifier; SAX applications and
+ /// extensions may use this class for other, similar purposes.
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <see cref="SAXNotSupportedException"/>
+ // LUCENENET: All exeption classes should be marked serializable
+#if FEATURE_SERIALIZABLE
+ [Serializable]
+#endif
+ public class SAXNotRecognizedException : SAXException
+ {
+ /// <summary>
+ /// Default constructor.
+ /// </summary>
+ public SAXNotRecognizedException()
+ : base()
+ {
+ }
+
+ /// <summary>
+ /// Construct a new exception with the given message.
+ /// </summary>
+ /// <param name="message">The text message of the exception.</param>
+ public SAXNotRecognizedException(string message)
+ : base(message)
+ {
+ }
+
+#if FEATURE_SERIALIZABLE
+ /// <summary>
+ /// Initializes a new instance of this class with serialized data.
+ /// </summary>
+ /// <param name="info">The <see cref="SerializationInfo"/> that holds the serialized object data about the exception being thrown.</param>
+ /// <param name="context">The <see cref="StreamingContext"/> that contains contextual information about the source or destination.</param>
+ public SAXNotRecognizedException(SerializationInfo info, StreamingContext context)
+ : base(info, context)
+ {
+ }
+#endif
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/SAXNotSupportedException.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/SAXNotSupportedException.cs b/src/Lucene.Net.Benchmark/Support/Sax/SAXNotSupportedException.cs
new file mode 100644
index 0000000..d5184c9
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/SAXNotSupportedException.cs
@@ -0,0 +1,67 @@
+// SAXNotSupportedException.java - unsupported feature or value.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the Public Domain.
+// $Id: SAXNotSupportedException.java,v 1.7 2002/01/30 21:13:48 dbrownell Exp $
+
+using System;
+#if FEATURE_SERIALIZABLE
+using System.Runtime.Serialization;
+#endif
+
+namespace Sax
+{
+ /// <summary>
+ /// Exception class for an unsupported operation.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// An XMLReader will throw this exception when it recognizes a
+ /// feature or property identifier, but cannot perform the requested
+ /// operation(setting a state or value). Other SAX2 applications and
+ /// extensions may use this class for similar purposes.
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="SAXNotRecognizedException"/>
+ // LUCENENET: All exeption classes should be marked serializable
+#if FEATURE_SERIALIZABLE
+ [Serializable]
+#endif
+ public class SAXNotSupportedException : SAXException
+ {
+ /// <summary>
+ /// Construct a new exception with no message.
+ /// </summary>
+ public SAXNotSupportedException()
+ : base()
+ {
+ }
+
+ /// <summary>
+ /// Construct a new exception with the given message.
+ /// </summary>
+ /// <param name="message">The text message of the exception.</param>
+ public SAXNotSupportedException(string message)
+ : base(message)
+ {
+ }
+
+#if FEATURE_SERIALIZABLE
+ /// <summary>
+ /// Initializes a new instance of this class with serialized data.
+ /// </summary>
+ /// <param name="info">The <see cref="SerializationInfo"/> that holds the serialized object data about the exception being thrown.</param>
+ /// <param name="context">The <see cref="StreamingContext"/> that contains contextual information about the source or destination.</param>
+ public SAXNotSupportedException(SerializationInfo info, StreamingContext context)
+ : base(info, context)
+ {
+ }
+#endif
+ }
+}
[32/33] lucenenet git commit: lucene-cli: Added benchmark commands,
tests and documentation
Posted by ni...@apache.org.
lucene-cli: Added benchmark commands, tests and documentation
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/4a8507a1
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/4a8507a1
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/4a8507a1
Branch: refs/heads/master
Commit: 4a8507a1941503ce17823dce748ace1bd953b89e
Parents: 564a292
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Mon Aug 7 00:14:41 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Mon Aug 7 00:14:41 2017 +0700
----------------------------------------------------------------------
.../ByTask/Programmatic/Sample.cs | 2 +-
.../Commands/Benchmark/BenchmarkCommandTest.cs | 50 ++++++
.../BenchmarkExtractReutersCommandTest.cs | 61 +++++++
.../BenchmarkExtractWikipediaCommandTest.cs | 66 +++++++
.../BenchmarkFindQualityQueriesCommandTest.cs | 65 +++++++
.../Benchmark/BenchmarkRunCommandTest.cs | 60 +++++++
.../BenchmarkRunTrecEvalCommandTest.cs | 63 +++++++
.../lucene-cli/Resources/Strings.Designer.cs | 171 +++++++++++++++++++
src/tools/lucene-cli/Resources/Strings.resx | 57 +++++++
src/tools/lucene-cli/commands/RootCommand.cs | 4 +-
.../commands/benchmark/BenchmarkCommand.cs | 51 ++++++
.../BenchmarkExtractReutersCommand.cs | 52 ++++++
.../BenchmarkExtractWikipediaCommand.cs | 72 ++++++++
.../BenchmarkFindQualityQueriesCommand.cs | 51 ++++++
.../BenchmarkRunTrecEvalCommand.cs | 78 +++++++++
.../benchmark-run/BenchmarkRunCommand.cs | 50 ++++++
.../benchmark-sample/BenchmarkSampleCommand.cs | 53 ++++++
.../commands/demo/DemoConfiguration.cs | 1 +
.../docs/benchmark/extract-reuters.md | 31 ++++
.../docs/benchmark/extract-wikipedia.md | 35 ++++
.../docs/benchmark/find-quality-queries.md | 27 +++
src/tools/lucene-cli/docs/benchmark/index.md | 14 ++
.../lucene-cli/docs/benchmark/run-trec-eval.md | 51 ++++++
src/tools/lucene-cli/docs/benchmark/run.md | 31 ++++
src/tools/lucene-cli/docs/benchmark/sample.md | 29 ++++
src/tools/lucene-cli/project.json | 2 +
26 files changed, 1223 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs b/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
index 6b248f6..3dbb07b 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs
@@ -72,7 +72,7 @@ namespace Lucene.Net.Benchmarks.ByTask.Programmatic
var p = new Dictionary<string, string>();
p["task.max.depth.log"] = "3";
p["max.buffered"] = "buf:10:10:100:100:10:10:100:100";
- p["doc.maker"] = "Lucene.Net.Benchmarks.ByTask.Feeds.ReutersContentSource, Lucene.Net.Benchmark";
+ //p["doc.maker"] = "Lucene.Net.Benchmarks.ByTask.Feeds.ReutersContentSource, Lucene.Net.Benchmark";
p["log.step"] = "2000";
p["doc.delete.step"] = "8";
p["analyzer"] = "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common";
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkCommandTest.cs
----------------------------------------------------------------------
diff --git a/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkCommandTest.cs b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkCommandTest.cs
new file mode 100644
index 0000000..999b282
--- /dev/null
+++ b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkCommandTest.cs
@@ -0,0 +1,50 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Cli.CommandLine;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli.Commands
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkCommandTest : CommandTestCase
+ {
+ protected override ConfigurationBase CreateConfiguration(MockConsoleApp app)
+ {
+ return new DemoCommand.Configuration(new CommandLineOptions()) { Main = (args) => app.Main(args) };
+ }
+
+ protected override IList<Arg[]> GetOptionalArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>();
+ }
+ protected override IList<Arg[]> GetRequiredArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>();
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestTooManyArguments()
+ {
+ Assert.Throws<CommandParsingException>(() => AssertConsoleOutput("one", ""));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractReutersCommandTest.cs
----------------------------------------------------------------------
diff --git a/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractReutersCommandTest.cs b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractReutersCommandTest.cs
new file mode 100644
index 0000000..7d0c947
--- /dev/null
+++ b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractReutersCommandTest.cs
@@ -0,0 +1,61 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Cli.CommandLine;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli.Commands
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkExtractReutersCommandTest : CommandTestCase
+ {
+ protected override ConfigurationBase CreateConfiguration(MockConsoleApp app)
+ {
+ return new BenchmarkExtractReutersCommand.Configuration(new CommandLineOptions()) { Main = (args) => app.Main(args) };
+ }
+
+ protected override IList<Arg[]> GetOptionalArgs()
+ {
+ return new List<Arg[]>();
+ }
+
+ protected override IList<Arg[]> GetRequiredArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>()
+ {
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp", output: new string[] { @"C:\lucene-temp" }) },
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp2", output: new string[] { @"C:\lucene-temp2" }) },
+ };
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestNotEnoughArguments()
+ {
+ AssertConsoleOutput("one", FromResource("NotEnoughArguments", 2));
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestTooManyArguments()
+ {
+ Assert.Throws<CommandParsingException>(() => AssertConsoleOutput("one two three", ""));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractWikipediaCommandTest.cs
----------------------------------------------------------------------
diff --git a/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractWikipediaCommandTest.cs b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractWikipediaCommandTest.cs
new file mode 100644
index 0000000..f5d3477
--- /dev/null
+++ b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkExtractWikipediaCommandTest.cs
@@ -0,0 +1,66 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Cli.CommandLine;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli.Commands
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkExtractWikipediaCommandTest : CommandTestCase
+ {
+ protected override ConfigurationBase CreateConfiguration(MockConsoleApp app)
+ {
+ return new BenchmarkExtractWikipediaCommand.Configuration(new CommandLineOptions()) { Main = (args) => app.Main(args) };
+ }
+
+ protected override IList<Arg[]> GetOptionalArgs()
+ {
+ return new List<Arg[]>()
+ {
+ new Arg[] { new Arg(inputPattern: "-d|--discard-image-only-docs", output: new string[] { "--discardImageOnlyDocs" }) },
+ };
+ }
+
+ protected override IList<Arg[]> GetRequiredArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>()
+ {
+ new Arg[] { new Arg(inputPattern: @"", output: new string[] { "--input" }) },
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp\docs.txt", output: new string[] { @"C:\lucene-temp\docs.txt" }) },
+ new Arg[] { new Arg(inputPattern: @"", output: new string[] { "--output" }) },
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp2", output: new string[] { @"C:\lucene-temp2" }) },
+ };
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestNotEnoughArguments()
+ {
+ AssertConsoleOutput("one", FromResource("NotEnoughArguments", 2));
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestTooManyArguments()
+ {
+ Assert.Throws<CommandParsingException>(() => AssertConsoleOutput("one two three", ""));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkFindQualityQueriesCommandTest.cs
----------------------------------------------------------------------
diff --git a/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkFindQualityQueriesCommandTest.cs b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkFindQualityQueriesCommandTest.cs
new file mode 100644
index 0000000..738040e
--- /dev/null
+++ b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkFindQualityQueriesCommandTest.cs
@@ -0,0 +1,65 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Cli.CommandLine;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli.Commands
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkFindQualityQueriesCommandTest : CommandTestCase
+ {
+ protected override ConfigurationBase CreateConfiguration(MockConsoleApp app)
+ {
+ return new BenchmarkFindQualityQueriesCommand.Configuration(new CommandLineOptions()) { Main = (args) => app.Main(args) };
+ }
+
+ protected override IList<Arg[]> GetOptionalArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>();
+ }
+
+ protected override IList<Arg[]> GetRequiredArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>()
+ {
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp", output: new string[] { @"C:\lucene-temp" }) },
+ };
+ }
+
+ /// <summary>
+ /// Ensures the current working directory is used when index directory is not supplied.
+ /// </summary>
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestNoArguments()
+ {
+ System.IO.Directory.SetCurrentDirectory(@"C:\");
+ AssertCommandTranslation("", new string[] { @"C:\" });
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestTooManyArguments()
+ {
+ Assert.Throws<CommandParsingException>(() => AssertConsoleOutput("one two", ""));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunCommandTest.cs
----------------------------------------------------------------------
diff --git a/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunCommandTest.cs b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunCommandTest.cs
new file mode 100644
index 0000000..ab87b70
--- /dev/null
+++ b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunCommandTest.cs
@@ -0,0 +1,60 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Cli.CommandLine;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli.Commands
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkRunCommandTest : CommandTestCase
+ {
+ protected override ConfigurationBase CreateConfiguration(MockConsoleApp app)
+ {
+ return new BenchmarkRunCommand.Configuration(new CommandLineOptions()) { Main = (args) => app.Main(args) };
+ }
+
+ protected override IList<Arg[]> GetOptionalArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>();
+ }
+ protected override IList<Arg[]> GetRequiredArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>()
+ {
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp\test.alg", output: new string[] { @"C:\lucene-temp\test.alg" }) },
+ };
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestNotEnoughArguments()
+ {
+ AssertConsoleOutput("", FromResource("NotEnoughArguments", 1));
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestTooManyArguments()
+ {
+ Assert.Throws<CommandParsingException>(() => AssertConsoleOutput("one two", ""));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunTrecEvalCommandTest.cs
----------------------------------------------------------------------
diff --git a/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunTrecEvalCommandTest.cs b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunTrecEvalCommandTest.cs
new file mode 100644
index 0000000..ba8a574
--- /dev/null
+++ b/src/tools/Lucene.Net.Tests.Cli/Commands/Benchmark/BenchmarkRunTrecEvalCommandTest.cs
@@ -0,0 +1,63 @@
+using Lucene.Net.Attributes;
+using Lucene.Net.Cli.CommandLine;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli.Commands
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkRunTrecEvalCommandTest : CommandTestCase
+ {
+ protected override ConfigurationBase CreateConfiguration(MockConsoleApp app)
+ {
+ return new BenchmarkRunTrecEvalCommand.Configuration(new CommandLineOptions()) { Main = (args) => app.Main(args) };
+ }
+
+ protected override IList<Arg[]> GetOptionalArgs()
+ {
+ return new List<Arg[]>();
+ }
+
+ protected override IList<Arg[]> GetRequiredArgs()
+ {
+ // NOTE: We must order this in the sequence of the expected output.
+ return new List<Arg[]>()
+ {
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp\input.txt", output: new string[] { @"C:\lucene-temp\input.txt" }) },
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp2\input.txt", output: new string[] { @"C:\lucene-temp2\input.txt" }) },
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp3\output.txt", output: new string[] { @"C:\lucene-temp3\output.txt" }) },
+ new Arg[] { new Arg(inputPattern: @"C:\lucene-temp4", output: new string[] { @"C:\lucene-temp4" }) },
+ };
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestNotEnoughArguments()
+ {
+ AssertConsoleOutput("one two three", FromResource("NotEnoughArguments", 4));
+ }
+
+ [Test]
+ [LuceneNetSpecific]
+ public virtual void TestTooManyArguments()
+ {
+ Assert.Throws<CommandParsingException>(() => AssertConsoleOutput("one two three four five", ""));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/Resources/Strings.Designer.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/Resources/Strings.Designer.cs b/src/tools/lucene-cli/Resources/Strings.Designer.cs
index 6cbbdde..caed9b1 100644
--- a/src/tools/lucene-cli/Resources/Strings.Designer.cs
+++ b/src/tools/lucene-cli/Resources/Strings.Designer.cs
@@ -207,6 +207,177 @@ namespace Lucene.Net.Cli.Resources {
}
/// <summary>
+ /// Looks up a localized string similar to Utilities for benchmarking Lucene.Net..
+ /// </summary>
+ public static string BenchmarkCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Splits Reuters SGML documents into simple text files containing: Title, Date, Dateline, Body..
+ /// </summary>
+ public static string BenchmarkExtractReutersCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractReutersCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Path to Reuters SGML files..
+ /// </summary>
+ public static string BenchmarkExtractReutersCommandInputDirectoryDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractReutersCommandInputDirectoryDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Path to a directory where the output files will be written..
+ /// </summary>
+ public static string BenchmarkExtractReutersCommandOutputDirectoryDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractReutersCommandOutputDirectoryDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Extracts a downloaded Wikipedia dump into separate files for indexing..
+ /// </summary>
+ public static string BenchmarkExtractWikipediaCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractWikipediaCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Tells the extractor to skip WIKI docs that contain only images..
+ /// </summary>
+ public static string BenchmarkExtractWikipediaCommandDiscardImageOnlyDocsDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractWikipediaCommandDiscardImageOnlyDocsDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Input path to a Wikipedia XML file..
+ /// </summary>
+ public static string BenchmarkExtractWikipediaCommandInputWikipediaFileDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractWikipediaCommandInputWikipediaFileDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Path to a directory where the output files will be written..
+ /// </summary>
+ public static string BenchmarkExtractWikipediaCommandOutputDirectoryDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkExtractWikipediaCommandOutputDirectoryDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Suggests quality queries based on index contents. Used for making quality test benchmarks..
+ /// </summary>
+ public static string BenchmarkFindQualityQueriesCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkFindQualityQueriesCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to An algorithm file specifying the parameters of the benchmark to run..
+ /// </summary>
+ public static string BenchmarkRunCommandAlgorithmFileDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunCommandAlgorithmFileDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Runs a benchmark based on an algorithm file..
+ /// </summary>
+ public static string BenchmarkRunCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Runs a TREC evaluation..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Output submission file for TREC evaluation..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandOutputSubmissionFileDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandOutputSubmissionFileDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Use description field in query..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandQueryOnDescriptionDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandQueryOnDescriptionDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Use narrative field in query..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandQueryOnNarrativeDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandQueryOnNarrativeDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Use title field in query. This flag will automatically be on if no other field is specified..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandQueryOnTitleDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandQueryOnTitleDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Input file conataining relevance judgements..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandQueryRelevanceFileDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandQueryRelevanceFileDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Input file containing queries..
+ /// </summary>
+ public static string BenchmarkRunTrecEvalCommandTopicsFileDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkRunTrecEvalCommandTopicsFileDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Sample of a performance test written programatically instead of using an algorithm file..
+ /// </summary>
+ public static string BenchmarkSampleCommandDescription {
+ get {
+ return ResourceManager.GetString("BenchmarkSampleCommandDescription", resourceCulture);
+ }
+ }
+
+ /// <summary>
/// Looks up a localized string similar to Cross check term vectors..
/// </summary>
public static string CrossCheckTermVectorsDescription {
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/Resources/Strings.resx
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/Resources/Strings.resx b/src/tools/lucene-cli/Resources/Strings.resx
index 4b1a488..592fe70 100644
--- a/src/tools/lucene-cli/Resources/Strings.resx
+++ b/src/tools/lucene-cli/Resources/Strings.resx
@@ -176,6 +176,63 @@ See this blog post (http://mentaldetritus.blogspot.com/2013/03/compiling-custom-
<data name="AnalysisStempelPatchStemsCommandStemmerTableFilesEncodingDescription" xml:space="preserve">
<value>The encoding to use for the stemmer table files. If not supplied, defaults to UTF-8.</value>
</data>
+ <data name="BenchmarkCommandDescription" xml:space="preserve">
+ <value>Utilities for benchmarking Lucene.Net.</value>
+ </data>
+ <data name="BenchmarkExtractReutersCommandDescription" xml:space="preserve">
+ <value>Splits Reuters SGML documents into simple text files containing: Title, Date, Dateline, Body.</value>
+ </data>
+ <data name="BenchmarkExtractReutersCommandInputDirectoryDescription" xml:space="preserve">
+ <value>Path to Reuters SGML files.</value>
+ </data>
+ <data name="BenchmarkExtractReutersCommandOutputDirectoryDescription" xml:space="preserve">
+ <value>Path to a directory where the output files will be written.</value>
+ </data>
+ <data name="BenchmarkExtractWikipediaCommandDescription" xml:space="preserve">
+ <value>Extracts a downloaded Wikipedia dump into separate files for indexing.</value>
+ </data>
+ <data name="BenchmarkExtractWikipediaCommandDiscardImageOnlyDocsDescription" xml:space="preserve">
+ <value>Tells the extractor to skip WIKI docs that contain only images.</value>
+ </data>
+ <data name="BenchmarkExtractWikipediaCommandInputWikipediaFileDescription" xml:space="preserve">
+ <value>Input path to a Wikipedia XML file.</value>
+ </data>
+ <data name="BenchmarkExtractWikipediaCommandOutputDirectoryDescription" xml:space="preserve">
+ <value>Path to a directory where the output files will be written.</value>
+ </data>
+ <data name="BenchmarkFindQualityQueriesCommandDescription" xml:space="preserve">
+ <value>Suggests quality queries based on index contents. Used for making quality test benchmarks.</value>
+ </data>
+ <data name="BenchmarkRunCommandAlgorithmFileDescription" xml:space="preserve">
+ <value>An algorithm file specifying the parameters of the benchmark to run.</value>
+ </data>
+ <data name="BenchmarkRunCommandDescription" xml:space="preserve">
+ <value>Runs a benchmark based on an algorithm file.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandDescription" xml:space="preserve">
+ <value>Runs a TREC evaluation.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandOutputSubmissionFileDescription" xml:space="preserve">
+ <value>Output submission file for TREC evaluation.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandQueryOnDescriptionDescription" xml:space="preserve">
+ <value>Use description field in query.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandQueryOnNarrativeDescription" xml:space="preserve">
+ <value>Use narrative field in query.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandQueryOnTitleDescription" xml:space="preserve">
+ <value>Use title field in query. This flag will automatically be on if no other field is specified.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandQueryRelevanceFileDescription" xml:space="preserve">
+ <value>Input file conataining relevance judgements.</value>
+ </data>
+ <data name="BenchmarkRunTrecEvalCommandTopicsFileDescription" xml:space="preserve">
+ <value>Input file containing queries.</value>
+ </data>
+ <data name="BenchmarkSampleCommandDescription" xml:space="preserve">
+ <value>Sample of a performance test written programatically instead of using an algorithm file.</value>
+ </data>
<data name="CrossCheckTermVectorsDescription" xml:space="preserve">
<value>Cross check term vectors.</value>
</data>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/RootCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/RootCommand.cs b/src/tools/lucene-cli/commands/RootCommand.cs
index c0d0b93..ff84328 100644
--- a/src/tools/lucene-cli/commands/RootCommand.cs
+++ b/src/tools/lucene-cli/commands/RootCommand.cs
@@ -25,10 +25,8 @@
{
this.Description = FromResource("RootCommandDescription");
- //// LUCENENET TODO: Fix this to use CommandLine stuff...
- //this.VersionOption("-v|--version", typeof(Program).GetTypeInfo().Assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>().InformationalVersion);
-
this.Commands.Add(new AnalysisCommand.Configuration(options));
+ this.Commands.Add(new BenchmarkCommand.Configuration(options));
this.Commands.Add(new IndexCommand.Configuration(options));
this.Commands.Add(new LockCommand.Configuration(options));
this.Commands.Add(new DemoCommand.Configuration(options));
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/BenchmarkCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/BenchmarkCommand.cs b/src/tools/lucene-cli/commands/benchmark/BenchmarkCommand.cs
new file mode 100644
index 0000000..73a71b3
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/BenchmarkCommand.cs
@@ -0,0 +1,51 @@
+using Lucene.Net.Benchmarks.ByTask;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkCommand : ICommand
+ {
+ public class Configuration : ConfigurationBase
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => Benchmark.Main(args);
+
+ this.Name = "benchmark";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+
+ this.Commands.Add(new BenchmarkExtractReutersCommand.Configuration(options));
+ this.Commands.Add(new BenchmarkExtractWikipediaCommand.Configuration(options));
+ this.Commands.Add(new BenchmarkFindQualityQueriesCommand.Configuration(options));
+ this.Commands.Add(new BenchmarkRunCommand.Configuration(options));
+ this.Commands.Add(new BenchmarkRunTrecEvalCommand.Configuration(options));
+ this.Commands.Add(new BenchmarkSampleCommand.Configuration(options));
+
+ this.OnExecute(() => new BenchmarkCommand().Run(this));
+ }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ cmd.ShowHelp();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/benchmark-extract-reuters/BenchmarkExtractReutersCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/benchmark-extract-reuters/BenchmarkExtractReutersCommand.cs b/src/tools/lucene-cli/commands/benchmark/benchmark-extract-reuters/BenchmarkExtractReutersCommand.cs
new file mode 100644
index 0000000..1cb226d
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/benchmark-extract-reuters/BenchmarkExtractReutersCommand.cs
@@ -0,0 +1,52 @@
+using Lucene.Net.Benchmarks.Utils;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkExtractReutersCommand : ICommand
+ {
+ public class Configuration : ConfigurationBase
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => ExtractReuters.Main(args);
+
+ this.Name = "extract-reuters";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+
+ this.Argument("<INPUT_DIRECTORY>", FromResource("InputDirectoryDescription"));
+ this.Argument("<OUTPUT_DIRECTORY>", FromResource("OutputDirectoryDescription"));
+
+ this.OnExecute(() => new BenchmarkExtractReutersCommand().Run(this));
+ }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ if (!cmd.ValidateArguments(2))
+ {
+ return 1;
+ }
+
+ cmd.Main(cmd.GetNonNullArguments());
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/benchmark-extract-wikipedia/BenchmarkExtractWikipediaCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/benchmark-extract-wikipedia/BenchmarkExtractWikipediaCommand.cs b/src/tools/lucene-cli/commands/benchmark/benchmark-extract-wikipedia/BenchmarkExtractWikipediaCommand.cs
new file mode 100644
index 0000000..c16fc7f
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/benchmark-extract-wikipedia/BenchmarkExtractWikipediaCommand.cs
@@ -0,0 +1,72 @@
+using Lucene.Net.Benchmarks.Utils;
+using Lucene.Net.Cli.CommandLine;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkExtractWikipediaCommand : ICommand
+ {
+ public class Configuration : ConfigurationBase
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => ExtractWikipedia.Main(args);
+
+ this.Name = "extract-wikipedia";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+
+ this.InputWikipediaFile = this.Argument("<INPUT_WIKIPEDIA_FILE>", FromResource("InputWikipediaFileDescription"));
+ this.OutputDirectory = this.Argument("<OUTPUT_DIRECTORY>", FromResource("OutputDirectoryDescription"));
+ this.DiscardImageOnlyDocs = this.Option("-d|--discard-image-only-docs", FromResource("DiscardImageOnlyDocsDescription"), CommandOptionType.NoValue);
+
+ this.OnExecute(() => new BenchmarkExtractWikipediaCommand().Run(this));
+ }
+
+ public CommandArgument InputWikipediaFile { get; set; }
+ public CommandArgument OutputDirectory { get; set; }
+ public CommandOption DiscardImageOnlyDocs { get; set; }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ if (!cmd.ValidateArguments(2))
+ {
+ return 1;
+ }
+
+ var args = new List<string>();
+ var input = cmd as Configuration;
+
+ args.Add("--input");
+ args.Add(input.InputWikipediaFile.Value);
+ args.Add("--output");
+ args.Add(input.OutputDirectory.Value);
+
+ if (input.DiscardImageOnlyDocs.HasValue())
+ {
+ args.Add("--discardImageOnlyDocs");
+ }
+
+ cmd.Main(args.ToArray());
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/benchmark-find-quality-queries/BenchmarkFindQualityQueriesCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/benchmark-find-quality-queries/BenchmarkFindQualityQueriesCommand.cs b/src/tools/lucene-cli/commands/benchmark/benchmark-find-quality-queries/BenchmarkFindQualityQueriesCommand.cs
new file mode 100644
index 0000000..81ad29b
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/benchmark-find-quality-queries/BenchmarkFindQualityQueriesCommand.cs
@@ -0,0 +1,51 @@
+using Lucene.Net.Benchmarks.Quality.Trec;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkFindQualityQueriesCommand : ICommand
+ {
+ public class Configuration : ConfigurationBase
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => QueryDriver.Main(args);
+
+ this.Name = "find-quality-queries";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+
+ this.Arguments.Add(new IndexDirectoryArgument(required: false));
+
+ this.OnExecute(() => new BenchmarkFindQualityQueriesCommand().Run(this));
+ }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ if (!cmd.ValidateArguments(1))
+ {
+ return 1;
+ }
+
+ cmd.Main(cmd.GetNonNullArguments());
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/benchmark-run-trec-eval/BenchmarkRunTrecEvalCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/benchmark-run-trec-eval/BenchmarkRunTrecEvalCommand.cs b/src/tools/lucene-cli/commands/benchmark/benchmark-run-trec-eval/BenchmarkRunTrecEvalCommand.cs
new file mode 100644
index 0000000..0d64231
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/benchmark-run-trec-eval/BenchmarkRunTrecEvalCommand.cs
@@ -0,0 +1,78 @@
+using Lucene.Net.Benchmarks.Quality.Trec;
+using Lucene.Net.Cli.CommandLine;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkRunTrecEvalCommand : ICommand
+ {
+ public class Configuration : ConfigurationBase
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => QueryDriver.Main(args);
+
+ this.Name = "run-trec-eval";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+
+ this.Argument("<INPUT_TOPICS_FILE>", FromResource("TopicsFileDescription"));
+ this.Argument("<INPUT_QUERY_RELEVANCE_FILE>", FromResource("QueryRelevanceFileDescription"));
+ this.Argument("<OUTPUT_SUBMISSION_FILE>", FromResource("OutputSubmissionFileDescription"));
+ this.Arguments.Add(new IndexDirectoryArgument(required: true));
+ this.QueryOnTitle = this.Option("-t|--query-on-title", FromResource("QueryOnTitleDescription"), CommandOptionType.NoValue);
+ this.QueryOnDescription = this.Option("-d|--query-on-description", FromResource("QueryOnDescriptionDescription"), CommandOptionType.NoValue);
+ this.QueryOnNarrative = this.Option("-n|--query-on-narrative", FromResource("QueryOnNarrativeDescription"), CommandOptionType.NoValue);
+
+ this.OnExecute(() => new BenchmarkRunTrecEvalCommand().Run(this));
+ }
+
+ public CommandOption QueryOnTitle { get; set; }
+ public CommandOption QueryOnDescription { get; set; }
+ public CommandOption QueryOnNarrative { get; set; }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ if (!cmd.ValidateArguments(4))
+ {
+ return 1;
+ }
+
+ var input = cmd as Configuration;
+ var args = new List<string>(cmd.GetNonNullArguments());
+
+ string querySpec = string.Empty;
+
+ if (input.QueryOnTitle.HasValue())
+ querySpec += "T";
+ if (input.QueryOnDescription.HasValue())
+ querySpec += "D";
+ if (input.QueryOnNarrative.HasValue())
+ querySpec += "N";
+
+ if (!string.IsNullOrEmpty(querySpec))
+ args.Add(querySpec);
+
+ cmd.Main(args.ToArray());
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/benchmark-run/BenchmarkRunCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/benchmark-run/BenchmarkRunCommand.cs b/src/tools/lucene-cli/commands/benchmark/benchmark-run/BenchmarkRunCommand.cs
new file mode 100644
index 0000000..4f29f6f
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/benchmark-run/BenchmarkRunCommand.cs
@@ -0,0 +1,50 @@
+using Lucene.Net.Benchmarks.ByTask;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkRunCommand : ICommand
+ {
+ public class Configuration : ConfigurationBase
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => Benchmark.Main(args);
+
+ this.Name = "run";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+ this.Argument("<ALGORITHM_FILE>", FromResource("AlgorithmFileDescription"));
+
+ this.OnExecute(() => new BenchmarkRunCommand().Run(this));
+ }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ if (!cmd.ValidateArguments(1))
+ {
+ return 1;
+ }
+
+ cmd.Main(cmd.GetNonNullArguments());
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/benchmark/benchmark-sample/BenchmarkSampleCommand.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/benchmark/benchmark-sample/BenchmarkSampleCommand.cs b/src/tools/lucene-cli/commands/benchmark/benchmark-sample/BenchmarkSampleCommand.cs
new file mode 100644
index 0000000..c8a3490
--- /dev/null
+++ b/src/tools/lucene-cli/commands/benchmark/benchmark-sample/BenchmarkSampleCommand.cs
@@ -0,0 +1,53 @@
+using Lucene.Net.Benchmarks.ByTask.Programmatic;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Cli
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class BenchmarkSampleCommand : ICommand
+ {
+ public class Configuration : DemoConfiguration
+ {
+ public Configuration(CommandLineOptions options)
+ {
+ this.Main = (args) => Sample.Main(args);
+
+ this.Name = "sample";
+ this.Description = FromResource("Description");
+ this.ExtendedHelpText = FromResource("ExtendedHelpText");
+
+ this.OnExecute(() => new BenchmarkSampleCommand().Run(this));
+ }
+
+ public override IEnumerable<string> SourceCodeFiles
+ {
+ get
+ {
+ return new string[] { "Sample.cs" };
+ }
+ }
+ }
+
+ public int Run(ConfigurationBase cmd)
+ {
+ cmd.Main(new string[0]);
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/commands/demo/DemoConfiguration.cs
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/commands/demo/DemoConfiguration.cs b/src/tools/lucene-cli/commands/demo/DemoConfiguration.cs
index d1f4eda..b2592c3 100644
--- a/src/tools/lucene-cli/commands/demo/DemoConfiguration.cs
+++ b/src/tools/lucene-cli/commands/demo/DemoConfiguration.cs
@@ -30,6 +30,7 @@ namespace Lucene.Net.Cli
protected DemoConfiguration()
{
+
this.viewSourceOption = this.Option(
"-src|--view-source-code",
Resources.Strings.ViewSourceCodeDescription,
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/extract-reuters.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/extract-reuters.md b/src/tools/lucene-cli/docs/benchmark/extract-reuters.md
new file mode 100644
index 0000000..d63b2d3
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/extract-reuters.md
@@ -0,0 +1,31 @@
+# extract-reuters
+
+### Name
+
+`benchmark-extract-reuters` - Splits Reuters SGML documents into simple text files containing: Title, Date, Dateline, Body.
+
+### Synopsis
+
+<code>dotnet lucene-cli.dll benchmark extract-reuters [?|-h|--help]</code>
+
+### Arguments
+
+`INPUT_DIRECTORY`
+
+Path to Reuters SGML files.
+
+`OUTPUT_DIRECTORY`
+
+Path to a directory where the output files will be written.
+
+### Options
+
+`?|-h|--help`
+
+Prints out a short help for the command.
+
+### Example
+
+Extracts the reuters SGML files in the `z:\input` directory and places the content in the `z:\output` directory.
+
+<code>dotnet lucene-cli.dll benchmark extract-reuters z:\input z:\output</code>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/extract-wikipedia.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/extract-wikipedia.md b/src/tools/lucene-cli/docs/benchmark/extract-wikipedia.md
new file mode 100644
index 0000000..1d48911
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/extract-wikipedia.md
@@ -0,0 +1,35 @@
+# extract-wikipedia
+
+### Name
+
+`benchmark-extract-wikipedia` - Extracts a downloaded Wikipedia dump into separate files for indexing.
+
+### Synopsis
+
+<code>dotnet lucene-cli.dll benchmark extract-wikipedia [?|-h|--help]</code>
+
+### Arguments
+
+`INPUT_WIKIPEDIA_FILE`
+
+Input path to a Wikipedia XML file.
+
+`OUTPUT_DIRECTORY`
+
+Path to a directory where the output files will be written.
+
+### Options
+
+`?|-h|--help`
+
+Prints out a short help for the command.
+
+`-d|--discard-image-only-docs`
+
+Tells the extractor to skip WIKI docs that contain only images.
+
+### Example
+
+Extracts the `c:\wiki.xml` file into the `c:\out` directory, skipping any docs that only contain images.
+
+<code>dotnet lucene-cli.dll benchmark extract-wikipedia c:\wiki.xml c:\out -d</code>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/find-quality-queries.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/find-quality-queries.md b/src/tools/lucene-cli/docs/benchmark/find-quality-queries.md
new file mode 100644
index 0000000..c4e615b
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/find-quality-queries.md
@@ -0,0 +1,27 @@
+# find-quality-queries
+
+### Name
+
+`benchmark-find-quality-queries` - Suggests quality queries based on index contents. Used for making quality test benchmarks.
+
+### Synopsis
+
+<code>dotnet lucene-cli.dll benchmark find-quality-queries [?|-h|--help]</code>
+
+### Arguments
+
+`INDEX_DIRECTORY`
+
+Path to the index.
+
+### Options
+
+`?|-h|--help`
+
+Prints out a short help for the command.
+
+### Example
+
+Finds quality queries on the `c:\lucene-index` index directory.
+
+<code>dotnet lucene-cli.dll benchmark find-quality-queries c:\lucene-index</code>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/index.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/index.md b/src/tools/lucene-cli/docs/benchmark/index.md
new file mode 100644
index 0000000..66d4e04
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/index.md
@@ -0,0 +1,14 @@
+# benchmark
+
+## Description
+
+Utilities for benchmarking Lucene.Net.
+
+## Commands
+
+- [extract-reuters](extract-reuters.md)
+- [extract-wikipedia](extract-wikipedia.md)
+- [find-quality-queries](find-quality-queries.md)
+- [run](run.md)
+- [run-trec-eval](run-trec-eval.md)
+- [sample](sample.md)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/run-trec-eval.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/run-trec-eval.md b/src/tools/lucene-cli/docs/benchmark/run-trec-eval.md
new file mode 100644
index 0000000..5d5960b
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/run-trec-eval.md
@@ -0,0 +1,51 @@
+# run-trec-eval
+
+### Name
+
+`benchmark-run-trec-eval` - Runs a TREC evaluation.
+
+### Synopsis
+
+<code>dotnet lucene-cli.dll benchmark run-trec-eval [?|-h|--help]</code>
+
+### Arguments
+
+`INPUT_TOPICS_FILE`
+
+Input file containing queries.
+
+`INPUT_QUERY_RELEVANCE_FILE`
+
+Input file conataining relevance judgements.
+
+`OUTPUT_SUBMISSION_FILE`
+
+Output submission file for TREC evaluation.
+
+`INDEX_DIRECTORY`
+
+The index directory.
+
+### Options
+
+`?|-h|--help`
+
+Prints out a short help for the command.
+
+`-t|--query-on-title`
+
+Use title field in query. This flag will automatically be on if no other field is specified.
+
+`-d|--query-on-description`
+
+Use description field in query.
+
+`-n|--query-on-narrative`
+
+Use narrative field in query.
+
+### Example
+
+Runs a TREC evaluation on the `c:\topics` queries file and the `c:\queries` relevance judgements on the `c:\lucene-index` index using the description and narrative fields and places the output in `c:\output.txt`.
+
+<code>dotnet lucene-cli.dll benchmark run-trec-eval c:\topics.txt c:\queries.txt c:\submissions.txt c:\output.txt c:\lucene-index -d -n</code>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/run.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/run.md b/src/tools/lucene-cli/docs/benchmark/run.md
new file mode 100644
index 0000000..8042307
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/run.md
@@ -0,0 +1,31 @@
+# run
+
+### Name
+
+`benchmark-run` - Runs a benchmark based on an algorithm file.
+
+### Synopsis
+
+<code>dotnet lucene-cli.dll benchmark run [?|-h|--help]</code>
+
+### Arguments
+
+`ALGORITHM_FILE`
+
+An algorithm file specifying the parameters of the benchmark to run.
+
+`OUTPUT_DIRECTORY`
+
+Path to a directory where the output files will be written.
+
+### Options
+
+`?|-h|--help`
+
+Prints out a short help for the command.
+
+### Example
+
+Runs a benchmark on the `c:\check.alg` algorithm file.
+
+<code>dotnet lucene-cli.dll benchmark run c:\check.alg</code>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/docs/benchmark/sample.md
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/docs/benchmark/sample.md b/src/tools/lucene-cli/docs/benchmark/sample.md
new file mode 100644
index 0000000..7245af5
--- /dev/null
+++ b/src/tools/lucene-cli/docs/benchmark/sample.md
@@ -0,0 +1,29 @@
+# sample
+
+### Name
+
+`benchmark-sample` - Sample of a performance test written programatically instead of using an algorithm file.
+
+### Synopsis
+
+<code>dotnet lucene-cli.dll benchmark sample [-src|--view-source-code] [-out|--output-source-code] [?|-h|--help]</code>
+
+### Options
+
+`?|-h|--help`
+
+Prints out a short help for the command.
+
+`-src|--view-source-code`
+
+Prints the source code to the console. Use `SPACE` or `n` to move to the next page of text, `ENTER` to scroll to the next line of text, `q` or `x` to quit.
+
+`-out|--output-source-code <DIRECTORY>`
+
+Outputs the source code to the specified directory.
+
+### Example
+
+Runs the sample.
+
+<code>dotnet lucene-cli.dll benchmark sample</code>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/4a8507a1/src/tools/lucene-cli/project.json
----------------------------------------------------------------------
diff --git a/src/tools/lucene-cli/project.json b/src/tools/lucene-cli/project.json
index 767a705..a33bdb4 100644
--- a/src/tools/lucene-cli/project.json
+++ b/src/tools/lucene-cli/project.json
@@ -10,6 +10,7 @@
},
"embed": {
"include": [
+ "../../Lucene.Net.Benchmark/ByTask/Programmatic/Sample.cs",
"../../Lucene.Net.Demo/*.cs",
"../../Lucene.Net.Demo/Facet/*.cs"
]
@@ -21,6 +22,7 @@
"Lucene.Net.Analysis.Common": "4.8.0",
"Lucene.Net.Analysis.Kuromoji": "4.8.0",
"Lucene.Net.Analysis.Stempel": "4.8.0",
+ "Lucene.Net.Benchmark": "4.8.0",
"Lucene.Net.Demo": "4.8.0",
"Lucene.Net.Expressions": "4.8.0",
"Lucene.Net.Facet": "4.8.0",
[21/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/SAXParseException.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/SAXParseException.cs b/src/Lucene.Net.Benchmark/Support/Sax/SAXParseException.cs
new file mode 100644
index 0000000..b7cdf64
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/SAXParseException.cs
@@ -0,0 +1,269 @@
+// SAX exception class.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: SAXParseException.java,v 1.11 2004/04/21 13:05:02 dmegginson Exp $
+
+using System;
+#if FEATURE_SERIALIZABLE
+using System.Runtime.Serialization;
+#endif
+
+namespace Sax
+{
+ /// <summary>
+ /// Encapsulate an XML parse error or warning.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This exception may include information for locating the error
+ /// in the original XML document, as if it came from a <see cref="ILocator"/>
+ /// object. Note that although the application
+ /// will receive a SAXParseException as the argument to the handlers
+ /// in the <see cref="IErrorHandler"/> interface,
+ /// the application is not actually required to throw the exception;
+ /// instead, it can simply read the information in it and take a
+ /// different action.
+ /// <para/>
+ /// Since this exception is a subclass of <see cref="SAXException"/>,
+ /// it inherits the ability to wrap another exception.
+ /// </remarks>
+ /// <since>SAX 1.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="SAXException"/>
+ /// <seealso cref="ILocator"/>
+ /// <seealso cref="IErrorHandler"/>
+#if FEATURE_SERIALIZABLE
+ [Serializable]
+#endif
+ public class SAXParseException : SAXException
+ {
+ //////////////////////////////////////////////////////////////////////
+ // Constructors.
+ //////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Construct a new exception with no message.
+ /// </summary>
+ // LUCENENET specific for serialization
+ public SAXParseException()
+ : base()
+ {
+ }
+
+ /// <summary>
+ /// Create a new <see cref="SAXParseException"/> from a message and a <see cref="ILocator"/>.
+ /// </summary>
+ /// <remarks>
+ /// This constructor is especially useful when an application is
+ /// creating its own exception from within a <see cref="IContentHandler"/>
+ /// callback.
+ /// </remarks>
+ /// <param name="message">The error or warning message.</param>
+ /// <param name="locator">The locator object for the error or warning (may be null).</param>
+ /// <seealso cref="ILocator"/>
+ public SAXParseException(string message, ILocator locator)
+ : base(message)
+ {
+ if (locator != null)
+ {
+ Init(locator.PublicId, locator.SystemId,
+ locator.LineNumber, locator.ColumnNumber);
+ }
+ else
+ {
+ Init(null, null, -1, -1);
+ }
+ }
+
+ /// <summary>
+ /// Wrap an existing exception in a SAXParseException.
+ /// </summary>
+ /// <remarks>
+ /// This constructor is especially useful when an application is
+ /// creating its own exception from within a <see cref="IContentHandler"/>
+ /// callback, and needs to wrap an existing exception that is not a
+ /// subclass of <see cref="SAXException"/>.
+ /// </remarks>
+ /// <param name="message">The error or warning message, or null to
+ /// use the message from the embedded exception.</param>
+ /// <param name="locator">The locator object for the error or warning (may be
+ /// null).</param>
+ /// <param name="e">Any exception.</param>
+ /// <seealso cref="ILocator"/>
+ public SAXParseException(string message, ILocator locator,
+ Exception e)
+ : base(message, e)
+ {
+ if (locator != null)
+ {
+ Init(locator.PublicId, locator.SystemId,
+ locator.LineNumber, locator.ColumnNumber);
+ }
+ else
+ {
+ Init(null, null, -1, -1);
+ }
+ }
+
+ /// <summary>
+ /// Create a new SAXParseException.
+ /// </summary>
+ /// <remarks>
+ /// This constructor is most useful for parser writers.
+ /// <para/>
+ /// All parameters except the message are as if
+ /// they were provided by a <see cref="ILocator"/>. For example, if the
+ /// system identifier is a URL (including relative filename), the
+ /// caller must resolve it fully before creating the exception.
+ /// </remarks>
+ /// <param name="message">The error or warning message.</param>
+ /// <param name="publicId">The public identifier of the entity that generated the error or warning.</param>
+ /// <param name="systemId">The system identifier of the entity that generated the error or warning.</param>
+ /// <param name="lineNumber">The line number of the end of the text that caused the error or warning.</param>
+ /// <param name="columnNumber">The column number of the end of the text that cause the error or warning.</param>
+ public SAXParseException(string message, string publicId, string systemId,
+ int lineNumber, int columnNumber)
+ : base(message)
+ {
+ Init(publicId, systemId, lineNumber, columnNumber);
+ }
+
+ /// <summary>
+ /// Create a new <see cref="SAXParseException"/> with an embedded exception.
+ /// </summary>
+ /// <remarks>
+ /// This constructor is most useful for parser writers who
+ /// need to wrap an exception that is not a subclass of
+ /// <see cref="SAXException"/>.
+ /// <para/>
+ /// All parameters except the message and exception are as if
+ /// they were provided by a <see cref="ILocator"/>. For example, if the
+ /// system identifier is a URL (including relative filename), the
+ /// caller must resolve it fully before creating the exception.
+ /// </remarks>
+ /// <param name="message">The error or warning message, or null to use the message from the embedded exception.</param>
+ /// <param name="publicId">The public identifier of the entity that generated the error or warning.</param>
+ /// <param name="systemId">The system identifier of the entity that generated the error or warning.</param>
+ /// <param name="lineNumber">The line number of the end of the text that caused the error or warning.</param>
+ /// <param name="columnNumber">The column number of the end of the text that cause the error or warning.</param>
+ /// <param name="e">Another exception to embed in this one.</param>
+ public SAXParseException(string message, string publicId, string systemId,
+ int lineNumber, int columnNumber, Exception e)
+ : base(message, e)
+ {
+ Init(publicId, systemId, lineNumber, columnNumber);
+ }
+
+#if FEATURE_SERIALIZABLE
+ /// <summary>
+ /// Initializes a new instance of this class with serialized data.
+ /// </summary>
+ /// <param name="info">The <see cref="SerializationInfo"/> that holds the serialized object data about the exception being thrown.</param>
+ /// <param name="context">The <see cref="StreamingContext"/> that contains contextual information about the source or destination.</param>
+ public SAXParseException(SerializationInfo info, StreamingContext context)
+ : base(info, context)
+ {
+ }
+#endif
+
+ /// <summary>
+ /// Internal initialization method.
+ /// </summary>
+ /// <param name="publicId">The public identifier of the entity which generated the exception, or null.</param>
+ /// <param name="systemId">The system identifier of the entity which generated the exception, or null.</param>
+ /// <param name="lineNumber">The line number of the error, or -1.</param>
+ /// <param name="columnNumber">The column number of the error, or -1.</param>
+ private void Init(string publicId, string systemId,
+ int lineNumber, int columnNumber)
+ {
+ this.publicId = publicId;
+ this.systemId = systemId;
+ this.lineNumber = lineNumber;
+ this.columnNumber = columnNumber;
+ }
+
+ /// <summary>
+ /// Get the public identifier of the entity where the exception occurred.
+ /// Returns a string containing the public identifier, or null if none is available.
+ /// </summary>
+ /// <seealso cref="ILocator.PublicId"/>
+ public string PublicId
+ {
+ get { return this.publicId; }
+ }
+
+ /// <summary>
+ /// Get the system identifier of the entity where the exception occurred.
+ /// <para/>
+ /// If the system identifier is a URL, it will have been resolved fully.
+ /// <para/>
+ /// A string containing the system identifier, or null if none is available.
+ /// </summary>
+ /// <seealso cref="ILocator.SystemId"/>
+ public string SystemId
+ {
+ get { return this.systemId; }
+ }
+
+ /// <summary>
+ /// The line number of the end of the text where the exception occurred.
+ /// <para/>
+ /// The first line is line 1.
+ /// <para/>
+ /// An integer representing the line number, or -1 if none is available.
+ /// </summary>
+ /// <seealso cref="ILocator.LineNumber"/>
+ public int LineNumber
+ {
+ get { return this.lineNumber; }
+ }
+
+ /// <summary>
+ /// The column number of the end of the text where the exception occurred.
+ /// <para/>
+ /// The first column in a line is position 1.
+ /// <para/>
+ /// An integer representing the column number, or -1
+ /// if none is available.
+ /// </summary>
+ /// <seealso cref="ILocator.ColumnNumber"/>
+ public int ColumnNumber
+ {
+ get { return this.columnNumber; }
+ }
+
+
+ //////////////////////////////////////////////////////////////////////
+ // Internal state.
+ //////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// The public identifier, or null.
+ /// </summary>
+ /// <seealso cref="PublicId"/>
+ private string publicId;
+
+ /// <summary>
+ /// The system identifier, or null.
+ /// </summary>
+ /// <seealso cref="SystemId"/>
+ private string systemId;
+
+ /// <summary>
+ /// The line number, or -1.
+ /// </summary>
+ /// <seealso cref="LineNumber"/>
+ private int lineNumber;
+
+ /// <summary>
+ /// The column number, or -1.
+ /// </summary>
+ /// <seealso cref="ColumnNumber"/>
+ private int columnNumber;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/XMLFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/XMLFilter.cs b/src/Lucene.Net.Benchmark/Support/Sax/XMLFilter.cs
new file mode 100644
index 0000000..f9350d3
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/XMLFilter.cs
@@ -0,0 +1,41 @@
+// XMLFilter.java - filter SAX2 events.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the Public Domain.
+// $Id: XMLFilter.java,v 1.6 2002/01/30 21:13:48 dbrownell Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Interface for an XML filter.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// An XML filter is like an XML reader, except that it obtains its
+ /// events from another XML reader rather than a primary source like
+ /// an XML document or database.Filters can modify a stream of
+ /// events as they pass on to the final application.
+ /// <para/>
+ /// The XMLFilterImpl helper class provides a convenient base
+ /// for creating SAX2 filters, by passing on all <see cref="IEntityResolver"/>, <see cref="IDTDHandler"/>,
+ /// <see cref="IContentHandler"/> and <see cref="IErrorHandler"/>
+ /// events automatically.
+ /// </remarks>
+ public interface IXMLFilter : IXMLReader
+ {
+ /// <summary>
+ /// Gets or sets the parent reader. Returns the parent filter, or null if none has been set.
+ /// </summary>
+ /// <remarks>
+ /// This method allows the application to link or query the parent
+ /// reader (which may be another filter). It is generally a
+ /// bad idea to perform any operations on the parent reader
+ /// directly: they should all pass through this filter.
+ /// </remarks>
+ IXMLReader Parent { get; set; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/XMLReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/XMLReader.cs b/src/Lucene.Net.Benchmark/Support/Sax/XMLReader.cs
new file mode 100644
index 0000000..71b690f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/XMLReader.cs
@@ -0,0 +1,305 @@
+// XMLFilter.java - filter SAX2 events.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the Public Domain.
+// $Id: XMLFilter.java,v 1.6 2002/01/30 21:13:48 dbrownell Exp $
+
+namespace Sax
+{
+ /// <summary>
+ /// Interface for an XML filter.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// An XML filter is like an XML reader, except that it obtains its
+ /// events from another XML reader rather than a primary source like
+ /// an XML document or database.Filters can modify a stream of
+ /// events as they pass on to the final application.
+ /// <para/>
+ /// The <see cref="IXMLFilter"/> helper class provides a convenient base
+ /// for creating SAX2 filters, by passing on all <see cref="IEntityResolver"/>,
+ /// <see cref="IDTDHandler"/>,
+ /// <see cref="IContentHandler"/> and <see cref="IErrorHandler"/>
+ /// events automatically.
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="Helpers.XMLFilter"/>
+ public interface IXMLReader
+ {
+ ////////////////////////////////////////////////////////////////////
+ // Configuration.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Look up the value of a feature flag.
+ /// </summary>
+ /// <remarks>
+ /// The feature name is any fully-qualified URI. It is
+ /// possible for an XMLReader to recognize a feature name but
+ /// temporarily be unable to return its value.
+ /// Some feature values may be available only in specific
+ /// contexts, such as before, during, or after a parse.
+ /// Also, some feature values may not be programmatically accessible.
+ /// (In the case of an adapter for SAX1 {@link Parser}, there is no
+ /// implementation-independent way to expose whether the underlying
+ /// parser is performing validation, expanding external entities,
+ /// and so forth.)
+ /// <para/>All XMLReaders are required to recognize the
+ /// http://xml.org/sax/features/namespaces and the
+ /// http://xml.org/sax/features/namespace-prefixes feature names.
+ /// <para/>Typical usage is something like this:
+ /// <code>
+ /// XMLReader r = new MySAXDriver();
+ /// // try to activate validation
+ /// try {
+ /// r.SetFeature("http://xml.org/sax/features/validation", true);
+ /// } catch (SAXException e) {
+ /// Console.Error.WriteLine("Cannot activate validation.");
+ /// }
+ /// // register event handlers
+ /// r.ContentHandler = new MyContentHandler();
+ /// r.ErrorHandler = new MyErrorHandler();
+ /// // parse the first document
+ /// try {
+ /// r.Parse("http://www.foo.com/mydoc.xml");
+ /// } catch (IOException e) {
+ /// Console.Error.WriteLine("I/O exception reading XML document");
+ /// } catch (SAXException e) {
+ /// Console.Error.WriteLine("XML exception reading document.");
+ /// }
+ /// </code>
+ /// <para/>Implementors are free (and encouraged) to invent their own features,
+ /// using names built on their own URIs.
+ /// </remarks>
+ /// <param name="name">The feature name, which is a fully-qualified URI.</param>
+ /// <returns>The current value of the feature (true or false).</returns>
+ /// <exception cref="SAXNotRecognizedException">If the feature
+ /// value can't be assigned or retrieved.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// <see cref="IXMLReader"/> recognizes the feature name but
+ /// cannot determine its value at this time.</exception>
+ /// <seealso cref="SetFeature(string, bool)"/>
+ bool GetFeature(string name);
+
+
+ /// <summary>
+ /// Set the value of a feature flag.
+ /// <para/>
+ /// The feature name is any fully-qualified URI. It is
+ /// possible for an XMLReader to expose a feature value but
+ /// to be unable to change the current value.
+ /// Some feature values may be immutable or mutable only
+ /// in specific contexts, such as before, during, or after
+ /// a parse.
+ /// <para/>
+ /// All XMLReaders are required to support setting
+ /// http://xml.org/sax/features/namespaces to true and
+ /// http://xml.org/sax/features/namespace-prefixes to false.
+ /// </summary>
+ /// <param name="name">The feature name, which is a fully-qualified URI.</param>
+ /// <param name="value">The requested value of the feature (true or false).</param>
+ /// <exception cref="SAXNotRecognizedException">If the feature
+ /// value can't be assigned or retrieved.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// <see cref="IXMLReader"/> recognizes the feature name but
+ /// cannot set the requested value.</exception>
+ /// <seealso cref="GetFeature(string)"/>
+ void SetFeature(string name, bool value);
+
+
+ /// <summary>
+ /// Look up the value of a property.
+ /// </summary>
+ /// <remarks>
+ /// The property name is any fully-qualified URI. It is
+ /// possible for an XMLReader to recognize a property name but
+ /// temporarily be unable to return its value.
+ /// Some property values may be available only in specific
+ /// contexts, such as before, during, or after a parse.
+ /// <para/>
+ /// <see cref="IXMLReader"/>s are not required to recognize any specific
+ /// property names, though an initial core set is documented for
+ /// SAX2.
+ /// <para/>
+ /// Implementors are free (and encouraged) to invent their own properties,
+ /// using names built on their own URIs.
+ /// </remarks>
+ /// <param name="name">The property name, which is a fully-qualified URI.</param>
+ /// <returns>The current value of the property.</returns>
+ /// <exception cref="SAXNotRecognizedException">If the property
+ /// value can't be assigned or retrieved.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// <see cref="IXMLReader"/> recognizes the property name but
+ /// cannot determine its value at this time.</exception>
+ /// <seealso cref="SetProperty(string, object)"/>
+ object GetProperty(string name);
+
+
+ /// <summary>
+ /// Set the value of a property.
+ /// </summary>
+ /// <remarks>
+ /// The property name is any fully-qualified URI. It is
+ /// possible for an <see cref="IXMLReader"/> to recognize a property name but
+ /// to be unable to change the current value.
+ /// Some property values may be immutable or mutable only
+ /// in specific contexts, such as before, during, or after
+ /// a parse.
+ /// <para/>
+ /// <see cref="IXMLReader"/>s are not required to recognize setting
+ /// any specific property names, though a core set is defined by
+ /// SAX2.
+ /// <para/>
+ /// This method is also the standard mechanism for setting
+ /// extended handlers.
+ /// </remarks>
+ /// <param name="name">The property name, which is a fully-qualified URI.</param>
+ /// <param name="value">The requested value for the property.</param>
+ /// <exception cref="SAXNotRecognizedException">If the property
+ /// value can't be assigned or retrieved.</exception>
+ /// <exception cref="SAXNotSupportedException">When the
+ /// <see cref="IXMLReader"/> recognizes the property name but
+ /// cannot set the requested value.</exception>
+ void SetProperty(string name, object value);
+
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Event handlers.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Gets or Sets an entity resolver.
+ /// </summary>
+ /// <remarks>
+ /// If the application does not register an entity resolver,
+ /// the <see cref="IXMLReader"/> will perform its own default resolution.
+ /// <para/>
+ /// Applications may register a new or different resolver in the
+ /// middle of a parse, and the SAX parser must begin using the new
+ /// resolver immediately.
+ /// </remarks>
+ IEntityResolver EntityResolver { get; set; }
+
+ /// <summary>
+ /// Gets or Sets a DTD event handler.
+ /// </summary>
+ /// <remarks>
+ /// If the application does not register a DTD handler, all DTD
+ /// events reported by the SAX parser will be silently ignored.
+ /// <para/>
+ /// Applications may register a new or different handler in the
+ /// middle of a parse, and the SAX parser must begin using the new
+ /// handler immediately.
+ /// </remarks>
+ IDTDHandler DTDHandler { get; set; }
+
+ /// <summary>
+ /// Gets or Sets a content event handler.
+ /// </summary>
+ /// <remarks>
+ /// <para/>If the application does not register a content handler, all
+ /// content events reported by the SAX parser will be silently
+ /// ignored.
+ /// <para/>Applications may register a new or different handler in the
+ /// middle of a parse, and the SAX parser must begin using the new
+ /// handler immediately.
+ /// </remarks>
+ IContentHandler ContentHandler { get; set; }
+
+
+ /// <summary>
+ /// Gets or Sets an error event handler.
+ /// </summary>
+ /// <remarks>
+ /// If the application does not register an error handler, all
+ /// error events reported by the SAX parser will be silently
+ /// ignored; however, normal processing may not continue. It is
+ /// highly recommended that all SAX applications implement an
+ /// error handler to avoid unexpected bugs.
+ /// <para/>
+ /// Applications may register a new or different handler in the
+ /// middle of a parse, and the SAX parser must begin using the new
+ /// handler immediately.
+ /// </remarks>
+ IErrorHandler ErrorHandler { get; set; }
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Parsing.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Parse an XML document.
+ /// </summary>
+ /// <remarks>
+ /// The application can use this method to instruct the XML
+ /// reader to begin parsing an XML document from any valid input
+ /// source (a character stream, a byte stream, or a URI).
+ /// <para/>
+ /// Applications may not invoke this method while a parse is in
+ /// progress (they should create a new XMLReader instead for each
+ /// nested XML document). Once a parse is complete, an
+ /// application may reuse the same XMLReader object, possibly with a
+ /// different input source.
+ /// Configuration of the <see cref="IXMLReader"/> object (such as handler bindings and
+ /// values established for feature flags and properties) is unchanged
+ /// by completion of a parse, unless the definition of that aspect of
+ /// the configuration explicitly specifies other behavior.
+ /// (For example, feature flags or properties exposing
+ /// characteristics of the document being parsed.)
+ /// <para/>
+ /// During the parse, the XMLReader will provide information
+ /// about the XML document through the registered event
+ /// handlers.
+ /// <para/>
+ /// This method is synchronous: it will not return until parsing
+ /// has ended. If a client application wants to terminate
+ /// parsing early, it should throw an exception.
+ /// </remarks>
+ /// <param name="input">The input source for the top-level of the
+ /// XML document.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <exception cref="System.IO.IOException">An IO exception from the parser,
+ /// possibly from a byte stream or character stream
+ /// supplied by the application.</exception>
+ /// <seealso cref="InputSource"/>
+ /// <seealso cref="Parse(string)"/>
+ /// <seealso cref="EntityResolver"/>
+ /// <seealso cref="DTDHandler"/>
+ /// <seealso cref="ContentHandler"/>
+ /// <seealso cref="ErrorHandler"/>
+ void Parse(InputSource input);
+
+
+ /// <summary>
+ /// Parse an XML document from a system identifier (URI).
+ /// </summary>
+ /// <remarks>
+ /// This method is a shortcut for the common case of reading a
+ /// document from a system identifier. It is the exact
+ /// equivalent of the following:
+ /// <code>
+ /// Parse(new InputSource(systemId));
+ /// </code>
+ /// <para/>If the system identifier is a URL, it must be fully resolved
+ /// by the application before it is passed to the parser.
+ /// </remarks>
+ /// <param name="input">The system identifier (URI).</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <exception cref="System.IO.IOException">An IO exception from the parser,
+ /// possibly from a byte stream or character stream
+ /// supplied by the application.</exception>
+ void Parse(string systemId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/StringExtensions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/StringExtensions.cs b/src/Lucene.Net.Benchmark/Support/StringExtensions.cs
new file mode 100644
index 0000000..2104fdb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/StringExtensions.cs
@@ -0,0 +1,14 @@
+namespace Lucene.Net.Support
+{
+ public static class StringExtensions
+ {
+ public static string Intern(this string value)
+ {
+#if NETSTANDARD
+ return value;
+#else
+ return string.Intern(value);
+#endif
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/AutoDetector.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/AutoDetector.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/AutoDetector.cs
new file mode 100644
index 0000000..6fcb578
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/AutoDetector.cs
@@ -0,0 +1,41 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+// Interface to objects that translate InputStreams to Readers by auto-detection
+
+using System.IO;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// Classes which accept an <see cref="Stream"/> and provide a <see cref="TextReader"/> which figures
+ /// out the encoding of the <see cref="Stream"/> and reads characters from it should
+ /// conform to this interface.
+ /// </summary>
+ /// <seealso cref="Stream" />
+ /// <seealso cref="TextReader" />
+ public interface IAutoDetector
+ {
+ /// <summary>
+ /// Given a <see cref="Stream"/>, return a suitable <see cref="TextReader"/> that understands
+ /// the presumed character encoding of that <see cref="Stream"/>.
+ /// If bytes are consumed from the <see cref="Stream"/> in the process, they
+ /// <i>must</i> be pushed back onto the InputStream so that they can be
+ /// reinterpreted as characters.
+ /// </summary>
+ /// <param name="stream">The <see cref="Stream"/></param>
+ /// <returns>A <see cref="TextReader"/> that reads from the <see cref="Stream"/></returns>
+ TextReader AutoDetectingReader(Stream stream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/Element.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/Element.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/Element.cs
new file mode 100644
index 0000000..dca7eed
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/Element.cs
@@ -0,0 +1,215 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+using Sax.Helpers;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// The internal representation of an actual element (not an element type).
+ /// An Element has an element type, attributes, and a successor Element
+ /// for use in constructing stacks and queues of Elements.
+ /// </summary>
+ /// <seealso cref="ElementType" />
+ /// <seealso cref="Sax.Net.Helpers.Attributes" />
+ public class Element
+ {
+ private readonly Attributes _atts; // attributes of element
+ private readonly ElementType _type; // type of element
+ private bool _preclosed; // this element has been preclosed
+
+ /// <summary>
+ /// Return an Element from a specified ElementType.
+ /// </summary>
+ /// <param name="type">
+ /// The element type of the newly constructed element
+ /// </param>
+ /// <param name="defaultAttributes">
+ /// True if default attributes are wanted
+ /// </param>
+ public Element(ElementType type, bool defaultAttributes)
+ {
+ _type = type;
+ if (defaultAttributes)
+ {
+ _atts = new Attributes(type.Attributes);
+ }
+ else
+ {
+ _atts = new Attributes();
+ }
+ Next = null;
+ _preclosed = false;
+ }
+
+ /// <summary>
+ /// Gets the element type.
+ /// </summary>
+ public virtual ElementType Type
+ {
+ get { return _type; }
+ }
+
+ /// <summary>
+ /// Gets the attributes as an Attributes object.
+ /// Returning an Attributes makes the attributes mutable.
+ /// </summary>
+ /// <seealso cref="Attributes" />
+ public virtual Attributes Attributes
+ {
+ get { return _atts; }
+ }
+
+ /// <summary>
+ /// Gets or sets the next element in an element stack or queue.
+ /// </summary>
+ public virtual Element Next { get; set; }
+
+ /// <summary>
+ /// Gets the name of the element's type.
+ /// </summary>
+ public virtual string Name
+ {
+ get { return _type.Name; }
+ }
+
+ /// <summary>
+ /// Gets the namespace name of the element's type.
+ /// </summary>
+ public virtual string Namespace
+ {
+ get { return _type.Namespace; }
+ }
+
+ /// <summary>
+ /// Gets the local name of the element's type.
+ /// </summary>
+ public virtual string LocalName
+ {
+ get { return _type.LocalName; }
+ }
+
+ /// <summary>
+ /// Gets the content model vector of the element's type.
+ /// </summary>
+ public virtual int Model
+ {
+ get { return _type.Model; }
+ }
+
+ /// <summary>
+ /// Gets the member-of vector of the element's type.
+ /// </summary>
+ public virtual int MemberOf
+ {
+ get { return _type.MemberOf; }
+ }
+
+ /// <summary>
+ /// Gets the flags vector of the element's type.
+ /// </summary>
+ public virtual int Flags
+ {
+ get { return _type.Flags; }
+ }
+
+ /// <summary>
+ /// Gets the parent element type of the element's type.
+ /// </summary>
+ public virtual ElementType Parent
+ {
+ get { return _type.Parent; }
+ }
+
+ /// <summary>
+ /// Return true if this element has been preclosed.
+ /// </summary>
+ public virtual bool IsPreclosed
+ {
+ get { return _preclosed; }
+ }
+
+ /// <summary>
+ /// Return true if the type of this element can contain the type of
+ /// another element.
+ /// Convenience method.
+ /// </summary>
+ /// <param name="other">
+ /// The other element
+ /// </param>
+ public virtual bool CanContain(Element other)
+ {
+ return _type.CanContain(other._type);
+ }
+
+ /// <summary>
+ /// Set an attribute and its value into this element.
+ /// </summary>
+ /// <param name="name">
+ /// The attribute name (Qname)
+ /// </param>
+ /// <param name="type">
+ /// The attribute type
+ /// </param>
+ /// <param name="value">
+ /// The attribute value
+ /// </param>
+ public virtual void SetAttribute(string name, string type, string value)
+ {
+ _type.SetAttribute(_atts, name, type, value);
+ }
+
+ /// <summary>
+ /// Make this element anonymous.
+ /// Remove any <c>id</c> or <c>name</c> attribute present
+ /// in the element's attributes.
+ /// </summary>
+ public virtual void Anonymize()
+ {
+ for (int i = _atts.Length - 1; i >= 0; i--)
+ {
+ if (_atts.GetType(i).Equals("ID") || _atts.GetQName(i).Equals("name"))
+ {
+ _atts.RemoveAttribute(i);
+ }
+ }
+ }
+
+ /// <summary>
+ /// Clean the attributes of this element.
+ /// Attributes with null name (the name was ill-formed)
+ /// or null value (the attribute was present in the element type but
+ /// not in this actual element) are removed.
+ /// </summary>
+ public virtual void Clean()
+ {
+ for (int i = _atts.Length - 1; i >= 0; i--)
+ {
+ string name = _atts.GetLocalName(i);
+ if (_atts.GetValue(i) == null || string.IsNullOrEmpty(name))
+ {
+ _atts.RemoveAttribute(i);
+ }
+ }
+ }
+
+ /// <summary>
+ /// Force this element to preclosed status, meaning that an end-tag has
+ /// been seen but the element cannot yet be closed for structural reasons.
+ /// </summary>
+ public virtual void Preclose()
+ {
+ _preclosed = true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
new file mode 100644
index 0000000..6d62a2f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/ElementType.cs
@@ -0,0 +1,269 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+using Sax.Helpers;
+using System;
+using System.Text;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// This class represents an element type in the schema.
+ /// An element type has a name, a content model vector, a member-of vector,
+ /// a flags vector, default attributes, and a schema to which it belongs.
+ /// </summary>
+ /// <seealso cref="Schema" />
+ public class ElementType
+ {
+ private readonly Attributes atts; // default attributes
+ private readonly string localName; // element type local name
+ private readonly string name; // element type name (Qname)
+ private readonly string @namespace; // element type namespace name
+ private readonly Schema schema; // schema to which this belongs
+
+ /// <summary>
+ /// Construct an <see cref="ElementType"/>:
+ /// but it's better to use <see cref="Schema.Element()"/> instead.
+ /// The content model, member-of, and flags vectors are specified as ints.
+ /// </summary>
+ /// <param name="name">The element type name</param>
+ /// <param name="model">ORed-together bits representing the content
+ /// models allowed in the content of this element type</param>
+ /// <param name="memberOf">ORed-together bits representing the content models
+ /// to which this element type belongs</param>
+ /// <param name="flags">ORed-together bits representing the flags associated
+ /// with this element type</param>
+ /// <param name="schema">
+ /// The schema with which this element type will be associated
+ /// </param>
+ public ElementType(string name, int model, int memberOf, int flags, Schema schema)
+ {
+ this.name = name;
+ Model = model;
+ MemberOf = memberOf;
+ Flags = flags;
+ atts = new Attributes();
+ this.schema = schema;
+ @namespace = GetNamespace(name, false);
+ localName = GetLocalName(name);
+ }
+
+ /// <summary>
+ /// Gets the name of this element type.
+ /// </summary>
+ public virtual string Name
+ {
+ get { return name; }
+ }
+
+ /// <summary>
+ /// Gets the namespace name of this element type.
+ /// </summary>
+ public virtual string Namespace
+ {
+ get { return @namespace; }
+ }
+
+ /// <summary>
+ /// Gets the local name of this element type.
+ /// </summary>
+ public virtual string LocalName
+ {
+ get { return localName; }
+ }
+
+ /// <summary>
+ /// Gets or sets the content models of this element type as a vector of bits
+ /// </summary>
+ public virtual int Model { get; set; }
+
+ /// <summary>
+ /// Gets or sets the content models to which this element type belongs as a vector of bits
+ /// </summary>
+ public virtual int MemberOf { get; set; }
+
+ /// <summary>
+ /// Gets or sets the flags associated with this element type as a vector of bits
+ /// </summary>
+ public virtual int Flags { get; set; }
+
+ /// <summary>
+ /// Returns the default attributes associated with this element type.
+ /// Attributes of type CDATA that don't have default values are
+ /// typically not included. Other attributes without default values
+ /// have an internal value of <c>null</c>.
+ /// The return value is an Attributes to allow the caller to mutate
+ /// the attributes.
+ /// </summary>
+ public virtual Attributes Attributes
+ {
+ get { return atts; }
+ }
+
+ /// <summary>
+ /// Gets or sets the parent element type of this element type.
+ /// </summary>
+ public virtual ElementType Parent { get; set; }
+
+ /// <summary>
+ /// Gets the schema which this element type is associated with.
+ /// </summary>
+ public virtual Schema Schema
+ {
+ get { return schema; }
+ }
+
+ /// <summary>
+ /// Return a namespace name from a Qname.
+ /// The attribute flag tells us whether to return an empty namespace
+ /// name if there is no prefix, or use the schema default instead.
+ /// </summary>
+ /// <param name="name">The Qname</param>
+ /// <param name="attribute">True if name is an attribute name</param>
+ /// <returns>The namespace name</returns>
+ public virtual string GetNamespace(string name, bool attribute)
+ {
+ int colon = name.IndexOf(':');
+ if (colon == -1)
+ {
+ return attribute ? "" : schema.Uri;
+ }
+ string prefix = name.Substring(0, colon);
+ if (prefix.Equals("xml"))
+ {
+ return "http://www.w3.org/XML/1998/namespace";
+ }
+ return string.Intern("urn:x-prefix:" + prefix);
+ }
+
+ /// <summary>
+ /// Return a local name from a Qname.
+ /// </summary>
+ /// <param name="name">The Qname</param>
+ /// <returns>The local name</returns>
+ public virtual string GetLocalName(string name)
+ {
+ int colon = name.IndexOf(':');
+ if (colon == -1)
+ {
+ return name;
+ }
+ return string.Intern(name.Substring(colon + 1));
+ }
+
+ /// <summary>
+ /// Returns <c>true</c> if this element type can contain another element type.
+ /// That is, if any of the models in this element's model vector
+ /// match any of the models in the other element type's member-of
+ /// vector.
+ /// </summary>
+ /// <param name="other">The other element type</param>
+ public virtual bool CanContain(ElementType other)
+ {
+ return (Model & other.MemberOf) != 0;
+ }
+
+ /// <summary>
+ /// Sets an attribute and its value into an <see cref="Sax.IAttributes"/> object.
+ /// Attempts to set a namespace declaration are ignored.
+ /// </summary>
+ /// <param name="atts">The <see cref="Sax.Helpers.Attributes"/> object</param>
+ /// <param name="name">The name (Qname) of the attribute</param>
+ /// <param name="type">The type of the attribute</param>
+ /// <param name="value">The value of the attribute</param>
+ public virtual void SetAttribute(Attributes atts, string name, string type, string value)
+ {
+ if (name.Equals("xmlns") || name.StartsWith("xmlns:"))
+ {
+ return;
+ }
+
+ string ns = GetNamespace(name, true);
+ string localName = GetLocalName(name);
+ int i = atts.GetIndex(name);
+ if (i == -1)
+ {
+ name = string.Intern(name);
+ if (type == null)
+ {
+ type = "CDATA";
+ }
+ if (!type.Equals("CDATA"))
+ {
+ value = Normalize(value);
+ }
+ atts.AddAttribute(ns, localName, name, type, value);
+ }
+ else
+ {
+ if (type == null)
+ {
+ type = atts.GetType(i);
+ }
+ if (!type.Equals("CDATA"))
+ {
+ value = Normalize(value);
+ }
+ atts.SetAttribute(i, ns, localName, name, type, value);
+ }
+ }
+
+ /// <summary>
+ /// Normalize an attribute value (ID-style).
+ /// CDATA-style attribute normalization is already done.
+ /// </summary>
+ /// <param name="value">The value to normalize</param>
+ public static string Normalize(string value)
+ {
+ if (value == null)
+ {
+ return null;
+ }
+ value = value.Trim();
+ if (value.IndexOf(" ", StringComparison.Ordinal) == -1)
+ {
+ return value;
+ }
+ bool space = false;
+ var b = new StringBuilder(value.Length);
+ foreach (char v in value)
+ {
+ if (v == ' ')
+ {
+ if (!space)
+ {
+ b.Append(v);
+ }
+ space = true;
+ }
+ else
+ {
+ b.Append(v);
+ space = false;
+ }
+ }
+ return b.ToString();
+ }
+
+ /// <summary>
+ /// Sets an attribute and its value into this element type.
+ /// </summary>
+ /// <param name="name">The name of the attribute</param>
+ /// <param name="type">The type of the attribute</param>
+ /// <param name="value">The value of the attribute</param>
+ public virtual void SetAttribute(string name, string type, string value)
+ {
+ SetAttribute(atts, name, type, value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLScanner.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLScanner.cs b/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLScanner.cs
new file mode 100644
index 0000000..ed41f84
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/TagSoup/HTMLScanner.cs
@@ -0,0 +1,745 @@
+// This file is part of TagSoup and is Copyright 2002-2008 by John Cowan.
+//
+// TagSoup is licensed under the Apache License,
+// Version 2.0. You may obtain a copy of this license at
+// http://www.apache.org/licenses/LICENSE-2.0 . You may also have
+// additional legal rights not granted by this license.
+//
+// TagSoup is distributed in the hope that it will be useful, but
+// unless required by applicable law or agreed to in writing, TagSoup
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied; not even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+//
+
+using Sax;
+using System;
+using System.IO;
+
+namespace TagSoup
+{
+ /// <summary>
+ /// This class implements a table-driven scanner for HTML, allowing for lots of
+ /// defects. It implements the Scanner interface, which accepts a Reader
+ /// object to fetch characters from and a ScanHandler object to report lexical
+ /// events to.
+ /// </summary>
+ public class HTMLScanner : IScanner, ILocator
+ {
+ // Start of state table
+ private const int S_ANAME = 1;
+ private const int S_APOS = 2;
+ private const int S_AVAL = 3;
+ private const int S_BB = 4;
+ private const int S_BBC = 5;
+ private const int S_BBCD = 6;
+ private const int S_BBCDA = 7;
+ private const int S_BBCDAT = 8;
+ private const int S_BBCDATA = 9;
+ private const int S_CDATA = 10;
+ private const int S_CDATA2 = 11;
+ private const int S_CDSECT = 12;
+ private const int S_CDSECT1 = 13;
+ private const int S_CDSECT2 = 14;
+ private const int S_COM = 15;
+ private const int S_COM2 = 16;
+ private const int S_COM3 = 17;
+ private const int S_COM4 = 18;
+ private const int S_DECL = 19;
+ private const int S_DECL2 = 20;
+ private const int S_DONE = 21;
+ private const int S_EMPTYTAG = 22;
+ private const int S_ENT = 23;
+ private const int S_EQ = 24;
+ private const int S_ETAG = 25;
+ private const int S_GI = 26;
+ private const int S_NCR = 27;
+ private const int S_PCDATA = 28;
+ private const int S_PI = 29;
+ private const int S_PITARGET = 30;
+ private const int S_QUOT = 31;
+ private const int S_STAGC = 32;
+ private const int S_TAG = 33;
+ private const int S_TAGWS = 34;
+ private const int S_XNCR = 35;
+ private const int A_ADUP = 1;
+ private const int A_ADUP_SAVE = 2;
+ private const int A_ADUP_STAGC = 3;
+ private const int A_ANAME = 4;
+ private const int A_ANAME_ADUP = 5;
+ private const int A_ANAME_ADUP_STAGC = 6;
+ private const int A_AVAL = 7;
+ private const int A_AVAL_STAGC = 8;
+ private const int A_CDATA = 9;
+ private const int A_CMNT = 10;
+ private const int A_DECL = 11;
+ private const int A_EMPTYTAG = 12;
+ private const int A_ENTITY = 13;
+ private const int A_ENTITY_START = 14;
+ private const int A_ETAG = 15;
+ private const int A_GI = 16;
+ private const int A_GI_STAGC = 17;
+ private const int A_LT = 18;
+ private const int A_LT_PCDATA = 19;
+ private const int A_MINUS = 20;
+ private const int A_MINUS2 = 21;
+ private const int A_MINUS3 = 22;
+ private const int A_PCDATA = 23;
+ private const int A_PI = 24;
+ private const int A_PITARGET = 25;
+ private const int A_PITARGET_PI = 26;
+ private const int A_SAVE = 27;
+ private const int A_SKIP = 28;
+ private const int A_SP = 29;
+ private const int A_STAGC = 30;
+ private const int A_UNGET = 31;
+ private const int A_UNSAVE_PCDATA = 32;
+ private static int[] statetable = {
+ S_ANAME, '/', A_ANAME_ADUP, S_EMPTYTAG,
+ S_ANAME, '=', A_ANAME, S_AVAL,
+ S_ANAME, '>', A_ANAME_ADUP_STAGC, S_PCDATA,
+ S_ANAME, 0, A_SAVE, S_ANAME,
+ S_ANAME, -1, A_ANAME_ADUP_STAGC, S_DONE,
+ S_ANAME, ' ', A_ANAME, S_EQ,
+ S_ANAME, '\n', A_ANAME, S_EQ,
+ S_ANAME, '\t', A_ANAME, S_EQ,
+ S_APOS, '\'', A_AVAL, S_TAGWS,
+ S_APOS, 0, A_SAVE, S_APOS,
+ S_APOS, -1, A_AVAL_STAGC, S_DONE,
+ S_APOS, ' ', A_SP, S_APOS,
+ S_APOS, '\n', A_SP, S_APOS,
+ S_APOS, '\t', A_SP, S_APOS,
+ S_AVAL, '\'', A_SKIP, S_APOS,
+ S_AVAL, '"', A_SKIP, S_QUOT,
+ S_AVAL, '>', A_AVAL_STAGC, S_PCDATA,
+ S_AVAL, 0, A_SAVE, S_STAGC,
+ S_AVAL, -1, A_AVAL_STAGC, S_DONE,
+ S_AVAL, ' ', A_SKIP, S_AVAL,
+ S_AVAL, '\n', A_SKIP, S_AVAL,
+ S_AVAL, '\t', A_SKIP, S_AVAL,
+ S_BB, 'C', A_SKIP, S_BBC,
+ S_BB, 0, A_SKIP, S_DECL,
+ S_BB, -1, A_SKIP, S_DONE,
+ S_BBC, 'D', A_SKIP, S_BBCD,
+ S_BBC, 0, A_SKIP, S_DECL,
+ S_BBC, -1, A_SKIP, S_DONE,
+ S_BBCD, 'A', A_SKIP, S_BBCDA,
+ S_BBCD, 0, A_SKIP, S_DECL,
+ S_BBCD, -1, A_SKIP, S_DONE,
+ S_BBCDA, 'T', A_SKIP, S_BBCDAT,
+ S_BBCDA, 0, A_SKIP, S_DECL,
+ S_BBCDA, -1, A_SKIP, S_DONE,
+ S_BBCDAT, 'A', A_SKIP, S_BBCDATA,
+ S_BBCDAT, 0, A_SKIP, S_DECL,
+ S_BBCDAT, -1, A_SKIP, S_DONE,
+ S_BBCDATA, '[', A_SKIP, S_CDSECT,
+ S_BBCDATA, 0, A_SKIP, S_DECL,
+ S_BBCDATA, -1, A_SKIP, S_DONE,
+ S_CDATA, '<', A_SAVE, S_CDATA2,
+ S_CDATA, 0, A_SAVE, S_CDATA,
+ S_CDATA, -1, A_PCDATA, S_DONE,
+ S_CDATA2, '/', A_UNSAVE_PCDATA, S_ETAG,
+ S_CDATA2, 0, A_SAVE, S_CDATA,
+ S_CDATA2, -1, A_UNSAVE_PCDATA, S_DONE,
+ S_CDSECT, ']', A_SAVE, S_CDSECT1,
+ S_CDSECT, 0, A_SAVE, S_CDSECT,
+ S_CDSECT, -1, A_SKIP, S_DONE,
+ S_CDSECT1, ']', A_SAVE, S_CDSECT2,
+ S_CDSECT1, 0, A_SAVE, S_CDSECT,
+ S_CDSECT1, -1, A_SKIP, S_DONE,
+ S_CDSECT2, '>', A_CDATA, S_PCDATA,
+ S_CDSECT2, 0, A_SAVE, S_CDSECT,
+ S_CDSECT2, -1, A_SKIP, S_DONE,
+ S_COM, '-', A_SKIP, S_COM2,
+ S_COM, 0, A_SAVE, S_COM2,
+ S_COM, -1, A_CMNT, S_DONE,
+ S_COM2, '-', A_SKIP, S_COM3,
+ S_COM2, 0, A_SAVE, S_COM2,
+ S_COM2, -1, A_CMNT, S_DONE,
+ S_COM3, '-', A_SKIP, S_COM4,
+ S_COM3, 0, A_MINUS, S_COM2,
+ S_COM3, -1, A_CMNT, S_DONE,
+ S_COM4, '-', A_MINUS3, S_COM4,
+ S_COM4, '>', A_CMNT, S_PCDATA,
+ S_COM4, 0, A_MINUS2, S_COM2,
+ S_COM4, -1, A_CMNT, S_DONE,
+ S_DECL, '-', A_SKIP, S_COM,
+ S_DECL, '[', A_SKIP, S_BB,
+ S_DECL, '>', A_SKIP, S_PCDATA,
+ S_DECL, 0, A_SAVE, S_DECL2,
+ S_DECL, -1, A_SKIP, S_DONE,
+ S_DECL2, '>', A_DECL, S_PCDATA,
+ S_DECL2, 0, A_SAVE, S_DECL2,
+ S_DECL2, -1, A_SKIP, S_DONE,
+ S_EMPTYTAG, '>', A_EMPTYTAG, S_PCDATA,
+ S_EMPTYTAG, 0, A_SAVE, S_ANAME,
+ S_EMPTYTAG, ' ', A_SKIP, S_TAGWS,
+ S_EMPTYTAG, '\n', A_SKIP, S_TAGWS,
+ S_EMPTYTAG, '\t', A_SKIP, S_TAGWS,
+ S_ENT, 0, A_ENTITY, S_ENT,
+ S_ENT, -1, A_ENTITY, S_DONE,
+ S_EQ, '=', A_SKIP, S_AVAL,
+ S_EQ, '>', A_ADUP_STAGC, S_PCDATA,
+ S_EQ, 0, A_ADUP_SAVE, S_ANAME,
+ S_EQ, -1, A_ADUP_STAGC, S_DONE,
+ S_EQ, ' ', A_SKIP, S_EQ,
+ S_EQ, '\n', A_SKIP, S_EQ,
+ S_EQ, '\t', A_SKIP, S_EQ,
+ S_ETAG, '>', A_ETAG, S_PCDATA,
+ S_ETAG, 0, A_SAVE, S_ETAG,
+ S_ETAG, -1, A_ETAG, S_DONE,
+ S_ETAG, ' ', A_SKIP, S_ETAG,
+ S_ETAG, '\n', A_SKIP, S_ETAG,
+ S_ETAG, '\t', A_SKIP, S_ETAG,
+ S_GI, '/', A_SKIP, S_EMPTYTAG,
+ S_GI, '>', A_GI_STAGC, S_PCDATA,
+ S_GI, 0, A_SAVE, S_GI,
+ S_GI, -1, A_SKIP, S_DONE,
+ S_GI, ' ', A_GI, S_TAGWS,
+ S_GI, '\n', A_GI, S_TAGWS,
+ S_GI, '\t', A_GI, S_TAGWS,
+ S_NCR, 0, A_ENTITY, S_NCR,
+ S_NCR, -1, A_ENTITY, S_DONE,
+ S_PCDATA, '&', A_ENTITY_START, S_ENT,
+ S_PCDATA, '<', A_PCDATA, S_TAG,
+ S_PCDATA, 0, A_SAVE, S_PCDATA,
+ S_PCDATA, -1, A_PCDATA, S_DONE,
+ S_PI, '>', A_PI, S_PCDATA,
+ S_PI, 0, A_SAVE, S_PI,
+ S_PI, -1, A_PI, S_DONE,
+ S_PITARGET, '>', A_PITARGET_PI, S_PCDATA,
+ S_PITARGET, 0, A_SAVE, S_PITARGET,
+ S_PITARGET, -1, A_PITARGET_PI, S_DONE,
+ S_PITARGET, ' ', A_PITARGET, S_PI,
+ S_PITARGET, '\n', A_PITARGET, S_PI,
+ S_PITARGET, '\t', A_PITARGET, S_PI,
+ S_QUOT, '"', A_AVAL, S_TAGWS,
+ S_QUOT, 0, A_SAVE, S_QUOT,
+ S_QUOT, -1, A_AVAL_STAGC, S_DONE,
+ S_QUOT, ' ', A_SP, S_QUOT,
+ S_QUOT, '\n', A_SP, S_QUOT,
+ S_QUOT, '\t', A_SP, S_QUOT,
+ S_STAGC, '>', A_AVAL_STAGC, S_PCDATA,
+ S_STAGC, 0, A_SAVE, S_STAGC,
+ S_STAGC, -1, A_AVAL_STAGC, S_DONE,
+ S_STAGC, ' ', A_AVAL, S_TAGWS,
+ S_STAGC, '\n', A_AVAL, S_TAGWS,
+ S_STAGC, '\t', A_AVAL, S_TAGWS,
+ S_TAG, '!', A_SKIP, S_DECL,
+ S_TAG, '/', A_SKIP, S_ETAG,
+ S_TAG, '?', A_SKIP, S_PITARGET,
+ S_TAG, '<', A_SAVE, S_TAG,
+ S_TAG, 0, A_SAVE, S_GI,
+ S_TAG, -1, A_LT_PCDATA, S_DONE,
+ S_TAG, ' ', A_LT, S_PCDATA,
+ S_TAG, '\n', A_LT, S_PCDATA,
+ S_TAG, '\t', A_LT, S_PCDATA,
+ S_TAGWS, '/', A_SKIP, S_EMPTYTAG,
+ S_TAGWS, '>', A_STAGC, S_PCDATA,
+ S_TAGWS, 0, A_SAVE, S_ANAME,
+ S_TAGWS, -1, A_STAGC, S_DONE,
+ S_TAGWS, ' ', A_SKIP, S_TAGWS,
+ S_TAGWS, '\n', A_SKIP, S_TAGWS,
+ S_TAGWS, '\t', A_SKIP, S_TAGWS,
+ S_XNCR, 0, A_ENTITY, S_XNCR,
+ S_XNCR, -1, A_ENTITY, S_DONE,
+
+ };
+ private static readonly string[] debug_actionnames = { "", "A_ADUP", "A_ADUP_SAVE", "A_ADUP_STAGC", "A_ANAME", "A_ANAME_ADUP", "A_ANAME_ADUP_STAGC", "A_AVAL", "A_AVAL_STAGC", "A_CDATA", "A_CMNT", "A_DECL", "A_EMPTYTAG", "A_ENTITY", "A_ENTITY_START", "A_ETAG", "A_GI", "A_GI_STAGC", "A_LT", "A_LT_PCDATA", "A_MINUS", "A_MINUS2", "A_MINUS3", "A_PCDATA", "A_PI", "A_PITARGET", "A_PITARGET_PI", "A_SAVE", "A_SKIP", "A_SP", "A_STAGC", "A_UNGET", "A_UNSAVE_PCDATA" };
+ private static readonly string[] debug_statenames = { "", "S_ANAME", "S_APOS", "S_AVAL", "S_BB", "S_BBC", "S_BBCD", "S_BBCDA", "S_BBCDAT", "S_BBCDATA", "S_CDATA", "S_CDATA2", "S_CDSECT", "S_CDSECT1", "S_CDSECT2", "S_COM", "S_COM2", "S_COM3", "S_COM4", "S_DECL", "S_DECL2", "S_DONE", "S_EMPTYTAG", "S_ENT", "S_EQ", "S_ETAG", "S_GI", "S_NCR", "S_PCDATA", "S_PI", "S_PITARGET", "S_QUOT", "S_STAGC", "S_TAG", "S_TAGWS", "S_XNCR" };
+
+ // End of state table
+
+ private string thePublicid; // Locator state
+ private string theSystemid;
+ private int theLastLine;
+ private int theLastColumn;
+ private int theCurrentLine;
+ private int theCurrentColumn;
+
+ int theState; // Current state
+ int theNextState; // Next state
+ char[] theOutputBuffer = new char[200]; // Output buffer
+ int theSize; // Current buffer size
+ int[] theWinMap = { // Windows chars map
+ 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
+ 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0x017D, 0xFFFD,
+ 0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
+ 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0x017E, 0x0178};
+
+ ///<summary>
+ /// Index into the state table for [state][input character - 2].
+ /// The state table consists of 4-entry runs on the form
+ /// { current state, input character, action, next state }.
+ /// We precompute the index into the state table for all possible
+ /// { current state, input character } and store the result in
+ /// the statetableIndex array. Since only some input characters
+ /// are present in the state table, we only do the computation for
+ /// characters 0 to the highest character value in the state table.
+ /// An input character of -2 is used to cover all other characters
+ /// as -2 is guaranteed not to match any input character entry
+ /// in the state table.
+ /// <para>When doing lookups, the input character should first be tested
+ /// to be in the range [-1 (inclusive), statetableIndexMaxChar (exclusive)].
+ /// if it isn't use -2 as the input character.
+ /// <para>Finally, add 2 to the input character to cover for the fact that
+ /// Java doesn't support negative array indexes. Then look up
+ /// the value in the statetableIndex. If the value is -1, then
+ /// no action or next state was found for the { state, input } that
+ /// you had. If it isn't -1, then action = statetable[value + 2] and
+ /// next state = statetable[value + 3]. That is, the value points
+ /// to the start of the answer 4-tuple in the statetable.
+ /// </summary>
+ static short[][] statetableIndex;
+
+ ///<summary>
+ /// The highest character value seen in the statetable.
+ /// See the doc comment for statetableIndex to see how this
+ /// is used.
+ /// </summary>
+ static int statetableIndexMaxChar;
+ public HTMLScanner()
+ {
+ int maxState = -1;
+ int maxChar = -1;
+ for (int i = 0; i < statetable.Length; i += 4)
+ {
+ if (statetable[i] > maxState)
+ {
+ maxState = statetable[i];
+ }
+ if (statetable[i + 1] > maxChar)
+ {
+ maxChar = statetable[i + 1];
+ }
+ }
+ statetableIndexMaxChar = maxChar + 1;
+
+ statetableIndex = new short[maxState + 1][];
+
+ for (int i = 0; i <= maxState; i++)
+ {
+ statetableIndex[i] = new short[maxChar + 3];
+ }
+ for (int theState = 0; theState <= maxState; ++theState)
+ {
+ for (int ch = -2; ch <= maxChar; ++ch)
+ {
+ int hit = -1;
+ int action = 0;
+ for (int i = 0; i < statetable.Length; i += 4)
+ {
+ if (theState != statetable[i])
+ {
+ if (action != 0) break;
+ continue;
+ }
+ if (statetable[i + 1] == 0)
+ {
+ hit = i;
+ action = statetable[i + 2];
+ }
+ else if (statetable[i + 1] == ch)
+ {
+ hit = i;
+ action = statetable[i + 2];
+ break;
+ }
+ }
+ statetableIndex[theState][ch + 2] = (short)hit;
+ }
+ }
+ }
+
+ // Locator implementation
+
+ public virtual int LineNumber
+ {
+ get { return theLastLine; }
+ }
+ public virtual int ColumnNumber
+ {
+ get { return theLastColumn; }
+ }
+ public virtual string PublicId
+ {
+ get { return thePublicid; }
+ }
+ public virtual string SystemId
+ {
+ get { return theSystemid; }
+ }
+
+
+ // Scanner implementation
+
+ /// <summary>
+ /// Reset document locator, supplying systemid and publicid.
+ /// </summary>
+ /// <param name="systemid">System id</param>
+ /// <param name="publicid">Public id</param>
+ public virtual void ResetDocumentLocator(string publicid, string systemid)
+ {
+ thePublicid = publicid;
+ theSystemid = systemid;
+ theLastLine = theLastColumn = theCurrentLine = theCurrentColumn = 0;
+ }
+
+ /// <summary>
+ /// Scan HTML source, reporting lexical events.
+ /// </summary>
+ /// <param name="r">Reader that provides characters</param>
+ /// <param name="h">ScanHandler that accepts lexical events.</param>
+ public virtual void Scan(TextReader r, IScanHandler h)
+ {
+ theState = S_PCDATA;
+
+ int firstChar = r.Peek(); // Remove any leading BOM
+ if (firstChar == '\uFEFF') r.Read();
+
+ while (theState != S_DONE)
+ {
+ int ch = r.Peek();
+ bool unread = false;
+
+ // Process control characters
+ if (ch >= 0x80 && ch <= 0x9F) ch = theWinMap[ch - 0x80];
+
+ if (ch == '\r')
+ {
+ r.Read();
+ ch = r.Peek(); // expect LF next
+ if (ch != '\n')
+ {
+ unread = true;
+ ch = '\n';
+ }
+ }
+
+ if (ch == '\n')
+ {
+ theCurrentLine++;
+ theCurrentColumn = 0;
+ }
+ else
+ {
+ theCurrentColumn++;
+ }
+
+ if (!(ch >= 0x20 || ch == '\n' || ch == '\t' || ch == -1)) continue;
+
+ // Search state table
+ int adjCh = (ch >= -1 && ch < statetableIndexMaxChar) ? ch : -2;
+ int statetableRow = statetableIndex[theState][adjCh + 2];
+ int action = 0;
+ if (statetableRow != -1)
+ {
+ action = statetable[statetableRow + 2];
+ theNextState = statetable[statetableRow + 3];
+ }
+
+ // System.err.println("In " + debug_statenames[theState] + " got " + nicechar(ch) + " doing " + debug_actionnames[action] + " then " + debug_statenames[theNextState]);
+ switch (action)
+ {
+ case 0:
+ throw new Exception(
+ "HTMLScanner can't cope with " + (int)ch + " in state " +
+ (int)theState);
+ case A_ADUP:
+ h.Adup(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_ADUP_SAVE:
+ h.Adup(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ Save(ch, h);
+ break;
+ case A_ADUP_STAGC:
+ h.Adup(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.STagC(theOutputBuffer, 0, theSize);
+ break;
+ case A_ANAME:
+ h.Aname(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_ANAME_ADUP:
+ h.Aname(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.Adup(theOutputBuffer, 0, theSize);
+ break;
+ case A_ANAME_ADUP_STAGC:
+ h.Aname(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.Adup(theOutputBuffer, 0, theSize);
+ h.STagC(theOutputBuffer, 0, theSize);
+ break;
+ case A_AVAL:
+ h.Aval(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_AVAL_STAGC:
+ h.Aval(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.STagC(theOutputBuffer, 0, theSize);
+ break;
+ case A_CDATA:
+ Mark();
+ // suppress the final "]]" in the buffer
+ if (theSize > 1) theSize -= 2;
+ h.PCDATA(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_ENTITY_START:
+ h.PCDATA(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ Save(ch, h);
+ break;
+ case A_ENTITY:
+ Mark();
+ char ch1 = (char)ch;
+ // System.out.println("Got " + ch1 + " in state " + ((theState == S_ENT) ? "S_ENT" : ((theState == S_NCR) ? "S_NCR" : "UNK")));
+ if (theState == S_ENT && ch1 == '#')
+ {
+ theNextState = S_NCR;
+ Save(ch, h);
+ break;
+ }
+ else if (theState == S_NCR && (ch1 == 'x' || ch1 == 'X'))
+ {
+ theNextState = S_XNCR;
+ Save(ch, h);
+ break;
+ }
+ else if (theState == S_ENT && char.IsLetterOrDigit(ch1))
+ {
+ Save(ch, h);
+ break;
+ }
+ else if (theState == S_NCR && char.IsDigit(ch1))
+ {
+ Save(ch, h);
+ break;
+ }
+ else if (theState == S_XNCR && (char.IsDigit(ch1) || "abcdefABCDEF".IndexOf(ch1) != -1))
+ {
+ Save(ch, h);
+ break;
+ }
+
+ // The whole entity reference has been collected
+ // System.err.println("%%" + new String(theOutputBuffer, 0, theSize));
+ h.Entity(theOutputBuffer, 1, theSize - 1);
+ int ent = h.GetEntity();
+ // System.err.println("%% value = " + ent);
+ if (ent != 0)
+ {
+ theSize = 0;
+ if (ent >= 0x80 && ent <= 0x9F)
+ {
+ ent = theWinMap[ent - 0x80];
+ }
+ if (ent < 0x20)
+ {
+ // Control becomes space
+ ent = 0x20;
+ }
+ else if (ent >= 0xD800 && ent <= 0xDFFF)
+ {
+ // Surrogates get dropped
+ ent = 0;
+ }
+ else if (ent <= 0xFFFF)
+ {
+ // BMP character
+ Save(ent, h);
+ }
+ else
+ {
+ // Astral converted to two surrogates
+ ent -= 0x10000;
+ Save((ent >> 10) + 0xD800, h);
+ Save((ent & 0x3FF) + 0xDC00, h);
+ }
+ if (ch != ';')
+ {
+ unread = true;
+ theCurrentColumn--;
+ }
+ }
+ else
+ {
+ unread = true;
+ theCurrentColumn--;
+ }
+ theNextState = S_PCDATA;
+ break;
+ case A_ETAG:
+ h.ETag(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_DECL:
+ h.Decl(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_GI:
+ h.GI(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_GI_STAGC:
+ h.GI(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.STagC(theOutputBuffer, 0, theSize);
+ break;
+ case A_LT:
+ Mark();
+ Save('<', h);
+ Save(ch, h);
+ break;
+ case A_LT_PCDATA:
+ Mark();
+ Save('<', h);
+ h.PCDATA(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_PCDATA:
+ Mark();
+ h.PCDATA(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_CMNT:
+ Mark();
+ h.Cmnt(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_MINUS3:
+ Save('-', h);
+ Save(' ', h);
+ break;
+ case A_MINUS2:
+ Save('-', h);
+ Save(' ', h);
+ Save('-', h);
+ Save(ch, h);
+ // fall through into A_MINUS
+ break;
+ case A_MINUS:
+ Save('-', h);
+ Save(ch, h);
+ break;
+ case A_PI:
+ Mark();
+ h.PI(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_PITARGET:
+ h.PITarget(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_PITARGET_PI:
+ h.PITarget(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.PI(theOutputBuffer, 0, theSize);
+ break;
+ case A_SAVE:
+ Save(ch, h);
+ break;
+ case A_SKIP:
+ break;
+ case A_SP:
+ Save(' ', h);
+ break;
+ case A_STAGC:
+ h.STagC(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ case A_EMPTYTAG:
+ Mark();
+ // System.err.println("%%% Empty tag seen");
+ if (theSize > 0) h.GI(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ h.STagE(theOutputBuffer, 0, theSize);
+ break;
+ case A_UNGET:
+ unread = true;
+ theCurrentColumn--;
+ break;
+ case A_UNSAVE_PCDATA:
+ if (theSize > 0) theSize--;
+ h.PCDATA(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ break;
+ default:
+ throw new Exception("Can't process state " + action);
+ }
+ if (!unread)
+ {
+ r.Read();
+ }
+ theState = theNextState;
+ }
+ h.EOF(theOutputBuffer, 0, 0);
+ }
+
+ /// <summary>
+ /// Mark the current scan position as a "point of interest" - start of a tag,
+ /// cdata, processing instruction etc.
+ /// </summary>
+ private void Mark()
+ {
+ theLastColumn = theCurrentColumn;
+ theLastLine = theCurrentLine;
+ }
+
+ /// <summary>
+ /// A callback for the ScanHandler that allows it to force
+ /// the lexer state to CDATA content (no markup is recognized except
+ /// the end of element.
+ /// </summary>
+ public virtual void StartCDATA() { theNextState = S_CDATA; }
+
+ private void Save(int ch, IScanHandler h)
+ {
+ if (theSize >= theOutputBuffer.Length - 20)
+ {
+ if (theState == S_PCDATA || theState == S_CDATA)
+ {
+ // Return a buffer-sized chunk of PCDATA
+ h.PCDATA(theOutputBuffer, 0, theSize);
+ theSize = 0;
+ }
+ else
+ {
+ // Grow the buffer size
+ char[] newOutputBuffer = new char[theOutputBuffer.Length * 2];
+ Array.Copy(theOutputBuffer, 0, newOutputBuffer, 0, theSize + 1);
+ theOutputBuffer = newOutputBuffer;
+ }
+ }
+ theOutputBuffer[theSize++] = (char)ch;
+ }
+
+ /**
+ Test procedure. Reads HTML from the standard input and writes
+ PYX to the standard output.
+ */
+
+ // public static void main(string[] argv) {
+ // IScanner s = new HTMLScanner();
+ // TextReader r = new StreamReader(System.in, "UTF-8");
+ // TextWriter w = new StreamWriter(System.out, "UTF-8");
+ // PYXWriter pw = new PYXWriter(w);
+ // s.scan(r, pw);
+ // w.close();
+ // }
+
+
+ private static string NiceChar(int value)
+ {
+ if (value == '\n') return "\\n";
+ if (value < 32) return "0x" + value.ToString("X");
+ return "'" + ((char)value) + "'";
+ }
+ }
+}
[23/33] lucenenet git commit: Lucene.Net.Benchmark: Added Sax and
TagSoup to the Support folder.
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/DeclHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/DeclHandler.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/DeclHandler.cs
new file mode 100644
index 0000000..047e9ac
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/DeclHandler.cs
@@ -0,0 +1,131 @@
+// DeclHandler.java - Optional handler for DTD declaration events.
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: DeclHandler.java,v 1.6 2004/04/22 13:28:49 dmegginson Exp $
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// SAX2 extension handler for DTD declaration events.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// See <a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This is an optional extension handler for SAX2 to provide more
+ /// complete information about DTD declarations in an XML document.
+ /// XML readers are not required to recognize this handler, and it
+ /// is not part of core-only SAX2 distributions.
+ /// <para/>
+ /// Note that data-related DTD declarations (unparsed entities and
+ /// notations) are already reported through the
+ /// <see cref="IDTDHandler"/> interface.
+ /// <para/>
+ /// If you are using the declaration handler together with a lexical
+ /// handler, all of the events will occur between the
+ /// <see cref="ILexicalHandler.StartDTD(string, string, string)"/> and the
+ /// <see cref="ILexicalHandler.EndDTD()"/> events.
+ /// <para/>
+ /// To set the DeclHandler for an XML reader, use the
+ /// <see cref="IXMLReader.SetProperty(string, object)"/> method
+ /// with the property name
+ /// <a href="http://xml.org/sax/properties/declaration-handler">http://xml.org/sax/properties/declaration-handler</a>
+ /// and an object implementing this interface (or null) as the value.
+ /// If the reader does not report declaration events, it will throw a
+ /// <see cref="SAXNotRecognizedException"/>
+ /// when you attempt to register the handler.
+ /// </remarks>
+ /// <since>SAX 2.0 (extensions 1.0)</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ public interface IDeclHandler
+ {
+ /// <summary>
+ /// Report an element type declaration.
+ /// </summary>
+ /// <remarks>
+ /// The content model will consist of the string "EMPTY", the
+ /// string "ANY", or a parenthesised group, optionally followed
+ /// by an occurrence indicator.The model will be normalized so
+ /// that all parameter entities are fully resolved and all whitespace
+ /// is removed,and will include the enclosing parentheses.Other
+ /// normalization (such as removing redundant parentheses or
+ /// simplifying occurrence indicators) is at the discretion of the
+ /// parser.
+ /// </remarks>
+ /// <param name="name">The element type name.</param>
+ /// <param name="model">The content model as a normalized string.</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ void ElementDecl(string name, string model);
+
+ /// <summary>
+ /// Report an attribute type declaration.
+ /// </summary>
+ /// <remarks>
+ /// Only the effective (first) declaration for an attribute will
+ /// be reported.The type will be one of the strings "CDATA",
+ /// "ID", "IDREF", "IDREFS", "NMTOKEN", "NMTOKENS", "ENTITY",
+ /// "ENTITIES", a parenthesized token group with
+ /// the separator "|" and all whitespace removed, or the word
+ /// "NOTATION" followed by a space followed by a parenthesized
+ /// token group with all whitespace removed.
+ /// <para/>
+ /// The value will be the value as reported to applications,
+ /// appropriately normalized and with entity and character
+ /// references expanded.
+ /// </remarks>
+ /// <param name="eName">The name of the associated element.</param>
+ /// <param name="aName">The name of the attribute.</param>
+ /// <param name="type">A string representing the attribute type.</param>
+ /// <param name="mode">A string representing the attribute defaulting mode
+ /// ("#IMPLIED", "#REQUIRED", or "#FIXED") or null if
+ /// none of these applies.</param>
+ /// <param name="value">A string representing the attribute's default value,
+ /// or null if there is none.</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ void AttributeDecl(string eName,
+ string aName,
+ string type,
+ string mode,
+ string value);
+
+ /// <summary>
+ /// Report an internal entity declaration.
+ /// </summary>
+ /// <remarks>
+ /// Only the effective (first) declaration for each entity
+ /// will be reported.All parameter entities in the value
+ /// will be expanded, but general entities will not.
+ /// </remarks>
+ /// <param name="name">The name of the entity. If it is a parameter
+ /// entity, the name will begin with '%'.</param>
+ /// <param name="value">The replacement text of the entity.</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="ExternalEntityDecl(string, string, string)"/>
+ /// <seealso cref="IDTDHandler.UnparsedEntityDecl(string, string, string, string)"/>
+ void InternalEntityDecl(string name, string value);
+
+ /// <summary>
+ /// Report a parsed external entity declaration.
+ /// </summary>
+ /// <remarks>
+ /// Only the effective (first) declaration for each entity
+ /// will be reported.
+ /// <para/>
+ /// If the system identifier is a URL, the parser must resolve it
+ /// fully before passing it to the application.
+ /// </remarks>
+ /// <param name="name">The name of the entity. If it is a parameter
+ /// entity, the name will begin with '%'.</param>
+ /// <param name="publicId">The entity's public identifier, or null if none
+ /// was given.</param>
+ /// <param name="systemId">The entity's system identifier.</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="InternalEntityDecl(string, string)"/>
+ /// <seealso cref="IDTDHandler.UnparsedEntityDecl(string, string, string, string)"/>
+ void ExternalEntityDecl(string name, string publicId,
+ string systemId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/DefaultHandler2.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/DefaultHandler2.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/DefaultHandler2.cs
new file mode 100644
index 0000000..409e44a
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/DefaultHandler2.cs
@@ -0,0 +1,112 @@
+// DefaultHandler2.java - extended DefaultHandler
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: DefaultHandler2.java,v 1.3 2002/01/12 19:04:19 dbrownell Exp $
+
+using Sax.Helpers;
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// This class extends the SAX2 base handler class to support the
+ /// SAX2 <see cref="ILexicalHandler"/>, <see cref="IDeclHandler"/>, and
+ /// <see cref="IEntityResolver2"/> extensions. Except for overriding the
+ /// original SAX1 <see cref="DefaultHandler.ResolveEntity(string, string)"/>
+ /// method the added handler methods just return. Subclassers may
+ /// override everything on a method-by-method basis.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// <para/>
+ /// <em>Note:</em> this class might yet learn that the
+ /// <see cref="IContentHandler.SetDocumentLocator(ILocator)"/> call might be passed a
+ /// <see cref="Locator2"/> object, and that the
+ /// <em>ContentHandler.startElement()</em> call might be passed a
+ /// <see cref="Attributes2"/> object.
+ /// </remarks>
+ /// <since>2.0 (extensions 1.1 alpha)</since>
+ /// <author>David Brownell</author>
+ /// <version>TBS</version>
+ public class DefaultHandler2 : DefaultHandler, ILexicalHandler, IDeclHandler, IEntityResolver2
+ {
+ /// <summary>Constructs a handler which ignores all parsing events.</summary>
+ public DefaultHandler2() { }
+
+
+ // SAX2 ext-1.0 LexicalHandler
+
+ public virtual void StartCDATA()
+ { }
+
+ public virtual void EndCDATA()
+ { }
+
+ public virtual void StartDTD(string name, string publicId, string systemId)
+ { }
+
+ public virtual void EndDTD()
+ { }
+
+ public virtual void StartEntity(string name)
+ { }
+
+ public virtual void EndEntity(string name)
+ { }
+
+ public virtual void Comment(char[] ch, int start, int length)
+ { }
+
+
+ // SAX2 ext-1.0 DeclHandler
+
+ public virtual void AttributeDecl(string eName, string aName,
+ string type, string mode, string value)
+ { }
+
+ public virtual void ElementDecl(string name, string model)
+ { }
+
+ public virtual void ExternalEntityDecl(string name,
+ string publicId, string systemId)
+ { }
+
+ public virtual void InternalEntityDecl(string name, string value)
+ { }
+
+ // SAX2 ext-1.1 EntityResolver2
+
+ /// <summary>
+ /// Tells the parser that if no external subset has been declared
+ /// in the document text, none should be used.
+ /// </summary>
+ public virtual InputSource GetExternalSubset(string name, string baseURI)
+ {
+ return null;
+ }
+
+ /// <summary>
+ /// Tells the parser to resolve the systemId against the baseURI
+ /// and read the entity text from that resulting absolute URI.
+ /// Note that because the older <see cref="DefaultHandler.ResolveEntity(string, string)"/>,
+ /// method is overridden to call this one, this method may sometimes
+ /// be invoked with null <paramref name="name"/> and <paramref name="baseURI"/>, and
+ /// with the <paramref name="systemId"/> already absolutized.
+ /// </summary>
+ public virtual InputSource ResolveEntity(string name, string publicId,
+ string baseURI, string systemId)
+ { return null; }
+
+ // SAX1 EntityResolver
+
+ /// <summary>
+ /// Invokes <see cref="IEntityResolver2.ResolveEntity(string, string, string, string)"/>
+ /// with null entity name and base URI.
+ /// You only need to override that method to use this class.
+ /// </summary>
+ public override InputSource ResolveEntity(string publicId, string systemId)
+ {
+ return ResolveEntity(null, publicId, null, systemId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/EntityResolver2.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/EntityResolver2.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/EntityResolver2.cs
new file mode 100644
index 0000000..4e836ed
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/EntityResolver2.cs
@@ -0,0 +1,178 @@
+// EntityResolver2.java - Extended SAX entity resolver.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: EntityResolver2.java,v 1.2 2002/01/12 19:20:08 dbrownell Exp $
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// Extended interface for mapping external entity references to input
+ /// sources, or providing a missing external subset. The
+ /// <see cref="IXMLReader.EntityResolver"/> property
+ /// is used to provide implementations of this interface to parsers.
+ /// When a parser uses the methods in this interface, the
+ /// <see cref="IEntityResolver2.ResolveEntity(string, string, string, string)"/>
+ /// method (in this interface) is used <em>instead of</em> the older (SAX 1.0)
+ /// <see cref="IEntityResolver.ResolveEntity(string, string)"/> method.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// <para/>
+ /// If a SAX application requires the customized handling which this
+ /// interface defines for external entities, it must ensure that it uses
+ /// an XMLReader with the
+ /// <em>http://xml.org/sax/features/use-entity-resolver2</em> feature flag
+ /// set to <em>true</em> (which is its default value when the feature is
+ /// recognized). If that flag is unrecognized, or its value is false,
+ /// or the resolver does not implement this interface, then only the
+ /// <see cref="IEntityResolver"/> method will be used.
+ /// <para/>
+ /// That supports three categories of application that modify entity
+ /// resolution. <em>Old Style</em> applications won't know about this interface;
+ /// they will provide an <see cref="IEntityResolver"/>.
+ /// <em>Transitional Mode</em> provide an <see cref="IEntityResolver2"/> and automatically
+ /// get the benefit of its methods in any systems (parsers or other tools)
+ /// supporting it, due to polymorphism.
+ /// Both <em>Old Style</em> and <em>Transitional Mode</em> applications will
+ /// work with any SAX2 parser.
+ /// <em>New style</em> applications will fail to run except on SAX2 parsers
+ /// that support this particular feature.
+ /// They will insist that feature flag have a value of "true", and the
+ /// <see cref="IEntityResolver2"/> implementation they provide might throw an exception
+ /// if the original SAX 1.0 style entity resolution method is invoked.
+ /// </remarks>
+ /// <seealso cref="IXMLReader.EntityResolver"/>
+ /// <since>SAX 2.0 (extensions 1.1 alpha)</since>
+ /// <author>David Brownell</author>
+ /// <version>TBD</version>
+ public interface IEntityResolver2 : IEntityResolver
+ {
+ /// <summary>
+ /// Allows applications to provide an external subset for documents
+ /// that don't explicitly define one. Documents with DOCTYPE declarations
+ /// that omit an external subset can thus augment the declarations
+ /// available for validation, entity processing, and attribute processing
+ /// (normalization, defaulting, and reporting types including ID).
+ /// This augmentation is reported
+ /// through the <see cref="ILexicalHandler.StartDTD(string, string, string)"/> method as if
+ /// the document text had originally included the external subset;
+ /// this callback is made before any internal subset data or errors
+ /// are reported.
+ /// <para/>
+ /// This method can also be used with documents that have no DOCTYPE
+ /// declaration.When the root element is encountered,
+ /// but no DOCTYPE declaration has been seen, this method is
+ /// invoked.If it returns a value for the external subset, that root
+ /// element is declared to be the root element, giving the effect of
+ /// splicing a DOCTYPE declaration at the end the prolog of a document
+ /// that could not otherwise be valid. The sequence of parser callbacks
+ /// in that case logically resembles this:
+ /// <para/>
+ /// <code>
+ /// ... comments and PIs from the prolog (as usual)
+ /// StartDTD("rootName", source.getPublicId (), source.getSystemId ());
+ /// StartEntity("[dtd]");
+ /// ... declarations, comments, and PIs from the external subset
+ /// EndEntity("[dtd]");
+ /// EndDTD();
+ /// ... then the rest of the document(as usual)
+ /// StartElement(..., "rootName", ...);
+ /// </code>
+ /// <para/>
+ /// Note that the InputSource gets no further resolution.
+ /// Implementations of this method may wish to invoke
+ /// <see cref="ResolveEntity(string, string, string, string)"/> to gain benefits such as use
+ /// of local caches of DTD entities.Also, this method will never be
+ /// used by a (non - validating) processor that is not including external
+ /// parameter entities.
+ /// <para/>
+ /// Uses for this method include facilitating data validation when
+ /// interoperating with XML processors that would always require
+ /// undesirable network accesses for external entities, or which for
+ /// other reasons adopt a "no DTDs" policy.
+ /// Non - validation motives include forcing documents to include DTDs so
+ /// that attributes are handled consistently.
+ /// For example, an XPath processor needs to know which attibutes have
+ /// type "ID" before it can process a widely used type of reference.
+ /// <para/>
+ /// <strong> Warning:</strong> Returning an external subset modifies
+ /// the input document.By providing definitions for general entities,
+ /// it can make a malformed document appear to be well formed.
+ /// </summary>
+ /// <param name="name">
+ /// Identifies the document root element. This name comes
+ /// from a DOCTYPE declaration (where available) or from the actual
+ /// root element.
+ /// </param>
+ /// <param name="baseURI">
+ /// The document's base URI, serving as an additional
+ /// hint for selecting the external subset. This is always an absolute
+ /// URI, unless it is null because the XMLReader was given an InputSource
+ /// without one.
+ /// </param>
+ /// <returns>An <see cref="InputSource"/> object describing the new external subset
+ /// to be used by the parser, or null to indicate that no external
+ /// subset is provided.
+ /// </returns>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping
+ /// another exception.</exception>
+ /// <exception cref="System.IO.IOException">Probably indicating a failure to create
+ /// a new <see cref="System.IO.Stream"/> or <see cref="System.IO.TextReader"/>, or an illegal URL.</exception>
+ InputSource GetExternalSubset(string name, string baseURI);
+
+ /// <summary>
+ /// Allows applications to map references to external entities into input
+ /// sources, or tell the parser it should use conventional URI resolution.
+ /// This method is only called for external entities which have been
+ /// properly declared.
+ /// <para/>
+ /// This method provides more flexibility than the <see cref="IEntityResolver"/>
+ /// interface, supporting implementations of more complex catalogue
+ /// schemes such as the one defined by the<a href="http://www.oasis-open.org/committees/entity/spec-2001-08-06.html">OASIS XML Catalogs</a> specification.
+ /// <para/>
+ /// Parsers configured to use this resolver method will call it
+ /// to determine the input source to use for any external entity
+ /// being included because of a reference in the XML text.
+ /// That excludes the document entity, and any external entity returned
+ /// by <see cref="GetExternalSubset(string, string)"/>.
+ /// When a(non - validating) processor is configured not to include
+ /// a class of entities(parameter or general) through use of feature
+ /// flags, this method is not invoked for such entities.
+ /// <para/>
+ /// Note that the entity naming scheme used here is the same one
+ /// used in the <see cref="ILexicalHandler"/>, or in the <see cref="IContentHandler.SkippedEntity(string)"/>
+ /// method.
+ /// </summary>
+ /// <param name="name">Identifies the external entity being resolved.
+ /// Either "[dtd]" for the external subset, or a name starting
+ /// with "%" to indicate a parameter entity, or else the name of
+ /// a general entity. This is never null when invoked by a SAX2</param>
+ /// <param name="publicId">The public identifier of the external entity being
+ /// referenced (normalized as required by the XML specification), or
+ /// null if none was supplied.</param>
+ /// <param name="baseURI">The URI with respect to which relative systemIDs
+ /// are interpreted. This is always an absolute URI, unless it is
+ /// null (likely because the <see cref="IXMLReader"/> was given an <see cref="InputSource"/> without
+ /// one). This URI is defined by the XML specification to be the one
+ /// associated with the "<" starting the relevant declaration.</param>
+ /// <param name="systemId">The system identifier of the external entity
+ /// being referenced; either a relative or absolute URI.
+ /// This is never null when invoked by a SAX2 parser; only declared
+ /// entities, and any external subset, are resolved by such parsers.</param>
+ /// <returns>An <see cref="InputSource"/> object describing the new input source to
+ /// be used by the parser. Returning null directs the parser to
+ /// resolve the system ID against the base URI and open a connection
+ /// to resulting URI.</returns>
+ /// <exception cref="SAXException">Any SAX exception, possibly wrapping
+ /// another exception.</exception>
+ /// <exception cref="System.IO.IOException">Probably indicating a failure to create
+ /// a new <see cref="System.IO.Stream"/> or <see cref="System.IO.TextReader"/>, or an illegal URL.</exception>
+ InputSource ResolveEntity(
+ string name,
+ string publicId,
+ string baseURI,
+ string systemId
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/LexicalHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/LexicalHandler.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/LexicalHandler.cs
new file mode 100644
index 0000000..4092f79
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/LexicalHandler.cs
@@ -0,0 +1,180 @@
+// LexicalHandler.java - optional handler for lexical parse events.
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: LexicalHandler.java,v 1.5 2002/01/30 21:00:44 dbrownell Exp $
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// SAX2 extension handler for lexical events.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This is an optional extension handler for SAX2 to provide
+ /// lexical information about an XML document, such as comments
+ /// and CDATA section boundaries.
+ /// XML readers are not required to recognize this handler, and it
+ /// is not part of core-only SAX2 distributions.
+ /// <para/>
+ /// The events in the lexical handler apply to the entire document,
+ /// not just to the document element, and all lexical handler events
+ /// must appear between the content handler's StartDocument and
+ /// EndDocument events.
+ /// <para/>
+ /// To set the LexicalHandler for an XML reader, use the
+ /// <see cref="IXMLReader.SetProperty(string, object)"/> method
+ /// with the property name
+ /// <a href="http://xml.org/sax/properties/lexical-handler">http://xml.org/sax/properties/lexical-handler</a>
+ /// and an object implementing this interface (or null) as the value.
+ /// If the reader does not report lexical events, it will throw a
+ /// <see cref="SAXNotRecognizedException"/>
+ /// when you attempt to register the handler.
+ /// </remarks>
+ /// <since>SAX 2.0 (extensions 1.0)</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ public interface ILexicalHandler
+ {
+ /// <summary>
+ /// Report the start of DTD declarations, if any.
+ /// </summary>
+ /// <remarks>
+ /// This method is intended to report the beginning of the
+ /// DOCTYPE declaration; if the document has no DOCTYPE declaration,
+ /// this method will not be invoked.
+ /// <para/>
+ /// All declarations reported through
+ /// <see cref="IDTDHandler"/> or
+ /// <see cref="Ext.IDeclHandler"/> events must appear
+ /// between the startDTD and <see cref="EndDTD()"/> events.
+ /// Declarations are assumed to belong to the internal DTD subset
+ /// unless they appear between <see cref="StartEntity(string)"/>
+ /// and <see cref="EndEntity(string)"/> events. Comments and
+ /// processing instructions from the DTD should also be reported
+ /// between the <see cref="StartDTD(string, string, string)"/> and <see cref="EndDTD()"/> events, in their original
+ /// order of(logical) occurrence; they are not required to
+ /// appear in their correct locations relative to <see cref="IDTDHandler"/>
+ /// or <see cref="IDeclHandler"/> events, however.
+ /// <para/>
+ /// Note that the start / endDTD events will appear within
+ /// the start / endDocument events from <see cref="IContentHandler"/> and
+ /// before the first <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/>
+ /// event.
+ /// </remarks>
+ /// <param name="name">The document type name.</param>
+ /// <param name="publicId">The declared public identifier for the
+ /// external DTD subset, or null if none was declared.</param>
+ /// <param name="systemId">The declared system identifier for the
+ /// external DTD subset, or null if none was declared.
+ /// (Note that this is not resolved against the document
+ /// base URI.)</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <see cref="EndDTD()"/>
+ /// <see cref="StartEntity(string)"/>
+ void StartDTD(string name, string publicId,
+ string systemId);
+
+ /// <summary>
+ /// Report the end of DTD declarations.
+ /// <para/>
+ /// This method is intended to report the end of the
+ /// DOCTYPE declaration; if the document has no DOCTYPE declaration,
+ /// this method will not be invoked.
+ /// </summary>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="StartDTD(string, string, string)"/>
+ void EndDTD();
+
+ /// <summary>
+ /// Report the beginning of some internal and external XML entities.
+ /// </summary>
+ /// <remarks>
+ /// The reporting of parameter entities (including
+ /// the external DTD subset) is optional, and SAX2 drivers that
+ /// report LexicalHandler events may not implement it; you can use the
+ /// <a href="http://xml.org/sax/features/lexical-handler/parameter-entities">http://xml.org/sax/features/lexical-handler/parameter-entities</a>
+ /// feature to query or control the reporting of parameter entities.
+ /// <para/>
+ /// General entities are reported with their regular names,
+ /// parameter entities have '%' prepended to their names, and
+ /// the external DTD subset has the pseudo-entity name "[dtd]".
+ /// <para/>
+ /// When a SAX2 driver is providing these events, all other
+ /// events must be properly nested within start/end entity
+ /// events. There is no additional requirement that events from
+ /// <see cref="IDeclHandler"/> or
+ /// <see cref="IDTDHandler"/> be properly ordered.
+ /// <para/>
+ /// Note that skipped entities will be reported through the
+ /// <see cref="IContentHandler.SkippedEntity(string)"/>
+ /// event, which is part of the ContentHandler interface.
+ /// <para/>Because of the streaming event model that SAX uses, some
+ /// entity boundaries cannot be reported under any
+ /// circumstances:
+ /// <list type="bullet">
+ /// <item><description>general entities within attribute values</description></item>
+ /// <item><description>parameter entities within declarations</description></item>
+ /// </list>
+ /// <para/>These will be silently expanded, with no indication of where
+ /// the original entity boundaries were.
+ /// <para/>Note also that the boundaries of character references (which
+ /// are not really entities anyway) are not reported.
+ /// <para/>All start/endEntity events must be properly nested.
+ /// </remarks>
+ /// <param name="name">The name of the entity. If it is a parameter
+ /// entity, the name will begin with '%', and if it is the
+ /// external DTD subset, it will be "[dtd]".</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="EndEntity(string)"/>
+ /// <seealso cref="IDeclHandler.InternalEntityDecl(string, string)"/>
+ /// <seealso cref="IDeclHandler.ExternalEntityDecl(string, string, string)"/>
+ void StartEntity(string name);
+
+ /// <summary>
+ /// Report the end of an entity.
+ /// </summary>
+ /// <param name="name">The name of the entity that is ending.</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="StartEntity(string)"/>
+ void EndEntity(string name);
+
+ /// <summary>
+ /// Report the start of a CDATA section.
+ /// </summary>
+ /// <remarks>
+ /// The contents of the CDATA section will be reported through
+ /// the regular <see cref="IContentHandler.Characters(char[], int, int)"/>
+ /// event; this event is intended only to report
+ /// the boundary.
+ /// </remarks>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="EndEntity(string)"/>
+ void StartCDATA();
+
+ /// <summary>
+ /// Report the end of a CDATA section.
+ /// </summary>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ /// <seealso cref="StartCDATA()"/>
+ void EndCDATA();
+
+ /// <summary>
+ /// Report an XML comment anywhere in the document.
+ /// <para/>
+ /// This callback will be used for comments inside or outside the
+ /// document element, including comments in the external DTD
+ /// subset(if read). Comments in the DTD must be properly
+ /// nested inside start/endDTD and start/endEntity events(if
+ /// used).
+ /// </summary>
+ /// <param name="ch">An array holding the characters in the comment.</param>
+ /// <param name="start">The starting position in the array.</param>
+ /// <param name="length">The number of characters to use from the array.</param>
+ /// <exception cref="SAXException">The application may raise an exception.</exception>
+ void Comment(char[] ch, int start, int length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2.cs
new file mode 100644
index 0000000..907d427
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2.cs
@@ -0,0 +1,64 @@
+// Locator2.java - extended Locator
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: Locator2.java,v 1.5 2004/03/17 14:30:10 dmegginson Exp $
+
+using System.Text;
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// SAX2 extension to augment the entity information provided
+ /// though a <see cref="ILocator"/>.
+ /// </summary>
+ /// <remarks>
+ /// If an implementation supports this extension, the Locator
+ /// provided in <see cref="IContentHandler.SetDocumentLocator(ILocator)"/>
+ /// will implement this interface, and the
+ /// <a href="http://xml.org/sax/features/use-locator2">http://xml.org/sax/features/use-locator2</a> feature
+ /// flag will have the value <em>true</em>.
+ /// <para/>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// <para/>
+ /// XMLReader implementations are not required to support this
+ /// information, and it is not part of core-only SAX2 distributions.
+ /// </remarks>
+ /// <since>SAX 2.0 (extensions 1.1 alpha)</since>
+ /// <author>David Brownell</author>
+ /// <version>TBS</version>
+ public interface ILocator2 : ILocator
+ {
+ /// <summary>
+ /// Returns the version of XML used for the entity. This will
+ /// normally be the identifier from the current entity's
+ /// <em><?xml version='...' ...?></em> declaration,
+ /// or be defaulted by the parser.
+ /// </summary>
+ string XMLVersion { get; }
+
+ /// <summary>
+ /// Returns the name of the character encoding for the entity.
+ /// If the encoding was declared externally(for example, in a MIME
+ /// Content-Type header), that will be the name returned.Else if there
+ /// was an<em><?xml ...encoding='...'?></em> declaration at
+ /// the start of the document, that encoding name will be returned.
+ /// Otherwise the encoding will been inferred (normally to be UTF-8, or
+ /// some UTF-16 variant), and that inferred name will be returned.
+ /// <para/>
+ /// When an <see cref="InputSource"/> is used
+ /// to provide an entity's character stream, this method returns the
+ /// encoding provided in that input stream.
+ /// <para/>
+ /// Note that some recent W3C specifications require that text
+ /// in some encodings be normalized, using Unicode Normalization
+ /// Form C, before processing.Such normalization must be performed
+ /// by applications, and would normally be triggered based on the
+ /// value returned by this method.
+ /// <para/>
+ /// Encoding names may be those used by the underlying JVM,
+ /// and comparisons should be case-insensitive.
+ /// </summary>
+ Encoding Encoding { get; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2Impl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2Impl.cs b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2Impl.cs
new file mode 100644
index 0000000..f4f460f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Ext/Locator2Impl.cs
@@ -0,0 +1,76 @@
+// Locator2Impl.java - extended LocatorImpl
+// http://www.saxproject.org
+// Public Domain: no warranty.
+// $Id: Locator2Impl.java,v 1.3 2004/04/26 17:34:35 dmegginson Exp $
+
+using Sax.Helpers;
+using System.Text;
+
+namespace Sax.Ext
+{
+ /// <summary>
+ /// SAX2 extension helper for holding additional Entity information,
+ /// implementing the <see cref="Locator2"/> interface.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with <strong>NO WARRANTY</strong>.</em>
+ /// <para/>
+ /// This is not part of core-only SAX2 distributions.
+ /// </remarks>
+ /// <since>SAX 2.0.2</since>
+ /// <author>David Brownell</author>
+ /// <version>TBS</version>
+ public class Locator2 : Locator, ILocator2
+ {
+ private Encoding encoding;
+ private string version;
+
+ /// <summary>
+ /// Construct a new, empty <see cref="Locator2"/> object.
+ /// This will not normally be useful, since the main purpose
+ /// of this class is to make a snapshot of an existing <see cref="Locator"/>.
+ /// </summary>
+ public Locator2() { }
+
+ /// <summary>
+ /// Copy an existing <see cref="Locator"/> or <see cref="Locator2"/> object.
+ /// If the object implements <see cref="Locator2"/>, values of the
+ /// <em>encoding</em> and <em>version</em>strings are copied,
+ /// otherwise they set to <em>null</em>.
+ /// </summary>
+ /// <param name="locator">The existing Locator object.</param>
+ public Locator2(ILocator locator)
+ : base(locator)
+ {
+ if (locator is Locator2) {
+ Locator2 l2 = (Locator2)locator;
+
+ version = l2.XMLVersion;
+ encoding = l2.Encoding;
+ }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Locator2 method implementations
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Gets the current value of the version property.
+ /// </summary>
+ public string XMLVersion
+ {
+ get { return version; }
+ set { version = value; }
+ }
+
+ /// <summary>
+ /// Gets the current value of the encoding property.
+ /// </summary>
+ public Encoding Encoding
+ {
+ get { return encoding; }
+ set { encoding = value; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Helpers/AttributesImpl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Helpers/AttributesImpl.cs b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/AttributesImpl.cs
new file mode 100644
index 0000000..53dc226
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/AttributesImpl.cs
@@ -0,0 +1,615 @@
+// AttributesImpl.java - default implementation of Attributes.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the public domain.
+// $Id: AttributesImpl.java,v 1.9 2002/01/30 20:52:24 dbrownell Exp $
+
+using System;
+
+namespace Sax.Helpers
+{
+ /// <summary>
+ /// Default implementation of the <see cref="Attributes"/> interface.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See <a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class provides a default implementation of the SAX2
+ /// <see cref="Attributes"/> interface, with the
+ /// addition of manipulators so that the list can be modified or
+ /// reused.
+ /// <para/>There are two typical uses of this class:
+ /// <list type="bullet">
+ /// <item><description>to take a persistent snapshot of an Attributes object
+ /// in a <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/> event; or</description></item>
+ /// <item><description>to construct or modify an Attributes object in a SAX2 driver or filter.</description></item>
+ /// </list>
+ /// <para/>
+ /// This class replaces the now-deprecated SAX1 AttributeListImpl
+ /// class; in addition to supporting the updated Attributes
+ /// interface rather than the deprecated IAttributeList
+ /// interface, it also includes a much more efficient
+ /// implementation using a single array rather than a set of Vectors.
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ public class Attributes : IAttributes
+ {
+ ////////////////////////////////////////////////////////////////////
+ // Constructors.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Construct a new, empty <see cref="Attributes"/> object.
+ /// </summary>
+ public Attributes()
+ {
+ length = 0;
+ data = null;
+ }
+
+ /// <summary>
+ /// Copy an existing Attributes object.
+ /// <para/>
+ /// This constructor is especially useful inside a
+ /// <see cref="IContentHandler.StartElement(string, string, string, IAttributes)"/>.
+ /// </summary>
+ /// <param name="atts">The existing <see cref="Attributes"/> object.</param>
+ public Attributes(IAttributes atts)
+ {
+ SetAttributes(atts);
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of org.xml.sax.Attributes.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Return the number of attributes in the list.
+ /// </summary>
+ /// <seealso cref="Attributes.Length"/>
+ public virtual int Length
+ {
+ get { return length; }
+ }
+
+ /// <summary>
+ /// Return an attribute's Namespace URI.
+ /// </summary>
+ /// <param name="index">The attribute's index (zero-based).</param>
+ /// <returns>The Namespace URI, the empty string if none is
+ /// available, or null if the index is out of range.</returns>
+ /// <seealso cref="Attributes.GetURI(int)"/>
+ public virtual string GetURI(int index)
+ {
+ if (index >= 0 && index < length)
+ {
+ return data[index * 5];
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+
+ /// <summary>
+ /// Return an attribute's local name.
+ /// </summary>
+ /// <param name="index">The attribute's index (zero-based).</param>
+ /// <returns>The attribute's local name, the empty string if none is available, or null if the index if out of range.</returns>
+ /// <seealso cref="Attributes.GetLocalName(int)"/>
+ public virtual string GetLocalName(int index)
+ {
+ if (index >= 0 && index < length)
+ {
+ return data[index * 5 + 1];
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+
+ /// <summary>
+ /// Return an attribute's qualified (prefixed) name.
+ /// </summary>
+ /// <param name="index">The attribute's index (zero-based).</param>
+ /// <returns>The attribute's qualified name, the empty string if
+ /// none is available, or null if the index is out of bounds.</returns>
+ /// <seealso cref="Attributes.GetQName(int)"/>
+ public virtual string GetQName(int index)
+ {
+ if (index >= 0 && index < length)
+ {
+ return data[index * 5 + 2];
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+
+ /// <summary>
+ /// Return an attribute's type by index.
+ /// </summary>
+ /// <param name="index">The attribute's index (zero-based).</param>
+ /// <returns>The attribute's type, "CDATA" if the type is unknown, or null
+ /// if the index is out of bounds.</returns>
+ /// <seealso cref="Attributes.GetType(int)"/>
+ public virtual string GetType(int index)
+ {
+ if (index >= 0 && index < length)
+ {
+ return data[index * 5 + 3];
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+
+ /// <summary>
+ /// Return an attribute's value by index.
+ /// </summary>
+ /// <param name="index">The attribute's index (zero-based).</param>
+ /// <returns>The attribute's value or null if the index is out of bounds.</returns>
+ /// <seealso cref="Attributes.GetValue(int)"/>
+ public virtual string GetValue(int index)
+ {
+ if (index >= 0 && index < length)
+ {
+ return data[index * 5 + 4];
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+
+ /// <summary>
+ /// Look up an attribute's index by Namespace name.
+ /// </summary>
+ /// <remarks>In many cases, it will be more efficient to look up the name once and
+ /// use the index query methods rather than using the name query methods
+ /// repeatedly.</remarks>
+ /// <param name="index">The attribute's Namespace URI, or the empty
+ /// string if none is available.</param>
+ /// <param name="localName">The attribute's local name.</param>
+ /// <returns>The attribute's index, or -1 if none matches.</returns>
+ /// <seealso cref="Attributes.GetIndex(string, string)"/>
+ public virtual int GetIndex(string uri, string localName)
+ {
+ int max = length * 5;
+ for (int i = 0; i < max; i += 5)
+ {
+ if (data[i].Equals(uri, StringComparison.Ordinal) && data[i + 1].Equals(localName, StringComparison.Ordinal))
+ {
+ return i / 5;
+ }
+ }
+ return -1;
+ }
+
+
+ /// <summary>
+ /// Look up an attribute's index by qualified (prefixed) name.
+ /// </summary>
+ /// <param name="qName">The qualified name.</param>
+ /// <returns>The attribute's index, or -1 if none matches.</returns>
+ /// <seealso cref="Attributes.GetIndex(string)"/>
+ public virtual int GetIndex(string qName)
+ {
+ int max = length * 5;
+ for (int i = 0; i < max; i += 5)
+ {
+ if (data[i + 2].Equals(qName, StringComparison.Ordinal))
+ {
+ return i / 5;
+ }
+ }
+ return -1;
+ }
+
+
+ /// <summary>
+ /// Look up an attribute's type by Namespace-qualified name.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty string for a name
+ /// with no explicit Namespace URI.</param>
+ /// <param name="localName">The local name.</param>
+ /// <returns>The attribute's type, or null if there is no matching attribute.</returns>
+ /// <seealso cref="Attributes.GetType(string, string)"/>
+ public virtual string GetType(string uri, string localName)
+ {
+ int max = length * 5;
+ for (int i = 0; i < max; i += 5)
+ {
+ if (data[i].Equals(uri, StringComparison.Ordinal) && data[i + 1].Equals(localName, StringComparison.Ordinal))
+ {
+ return data[i + 3];
+ }
+ }
+ return null;
+ }
+
+
+ /// <summary>
+ /// Look up an attribute's type by qualified (prefixed) name.
+ /// </summary>
+ /// <param name="qName">The qualified name.</param>
+ /// <returns>The attribute's type, or null if there is no
+ /// matching attribute.</returns>
+ /// <seealso cref="Attributes.GetType(string)"/>
+ public virtual string GetType(string qName)
+ {
+ int max = length * 5;
+ for (int i = 0; i < max; i += 5)
+ {
+ if (data[i + 2].Equals(qName, StringComparison.Ordinal))
+ {
+ return data[i + 3];
+ }
+ }
+ return null;
+ }
+
+
+ /// <summary>
+ /// Look up an attribute's value by Namespace-qualified name.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty string for a name
+ /// with no explicit Namespace URI.</param>
+ /// <param name="localName">The local name.</param>
+ /// <returns>The attribute's value, or null if there is no matching attribute.</returns>
+ /// <seealso cref="Attributes.GetValue(string, string)"/>
+ public virtual string GetValue(string uri, string localName)
+ {
+ int max = length * 5;
+ for (int i = 0; i < max; i += 5)
+ {
+ if (data[i].Equals(uri, StringComparison.Ordinal) && data[i + 1].Equals(localName, StringComparison.Ordinal))
+ {
+ return data[i + 4];
+ }
+ }
+ return null;
+ }
+
+
+ /// <summary>
+ /// Look up an attribute's value by qualified (prefixed) name.
+ /// </summary>
+ /// <param name="qName">The qualified name.</param>
+ /// <returns>The attribute's value, or null if there is no
+ /// matching attribute.</returns>
+ /// <seealso cref="Attributes.GetValue(string)"/>
+ public virtual string GetValue(string qName)
+ {
+ int max = length * 5;
+ for (int i = 0; i < max; i += 5)
+ {
+ if (data[i + 2].Equals(qName, StringComparison.Ordinal))
+ {
+ return data[i + 4];
+ }
+ }
+ return null;
+ }
+
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Manipulators.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Clear the attribute list for reuse.
+ /// <para/>
+ /// Note that little memory is freed by this call:
+ /// the current array is kept so it can be
+ /// reused.
+ /// <summary>
+ public virtual void Clear()
+ {
+ if (data != null)
+ {
+ for (int i = 0; i < (length * 5); i++)
+ data[i] = null;
+ }
+ length = 0;
+ }
+
+ /// <summary>
+ /// Copy an entire Attributes object.
+ /// <para/>
+ /// It may be more efficient to reuse an existing object
+ /// rather than constantly allocating new ones.
+ /// </summary>
+ /// <param name="atts">The attributes to copy.</param>
+ public virtual void SetAttributes(IAttributes atts)
+ {
+ Clear();
+ length = atts.Length;
+ if (length > 0)
+ {
+ data = new string[length * 5];
+ for (int i = 0; i < length; i++)
+ {
+ data[i * 5] = atts.GetURI(i);
+ data[i * 5 + 1] = atts.GetLocalName(i);
+ data[i * 5 + 2] = atts.GetQName(i);
+ data[i * 5 + 3] = atts.GetType(i);
+ data[i * 5 + 4] = atts.GetValue(i);
+ }
+ }
+ }
+
+
+ /// <summary>
+ /// Add an attribute to the end of the list.
+ /// <para/>For the sake of speed, this method does no checking
+ /// to see if the attribute is already in the list: that is
+ /// the responsibility of the application.
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty string if
+ /// none is available or Namespace processing is not
+ /// being performed.</param>
+ /// <param name="localName">The local name, or the empty string if
+ /// Namespace processing is not being performed.</param>
+ /// <param name="qName">The qualified (prefixed) name, or the empty string
+ /// if qualified names are not available.</param>
+ /// <param name="type">The attribute type as a string.</param>
+ /// <param name="value">The attribute value.</param>
+ public virtual void AddAttribute(string uri, string localName, string qName,
+ string type, string value)
+ {
+ EnsureCapacity(length + 1);
+ data[length * 5] = uri;
+ data[length * 5 + 1] = localName;
+ data[length * 5 + 2] = qName;
+ data[length * 5 + 3] = type;
+ data[length * 5 + 4] = value;
+ length++;
+ }
+
+
+ /// <summary>
+ /// Set an attribute in the list.
+ ///
+ /// <para/>For the sake of speed, this method does no checking
+ /// for name conflicts or well-formedness: such checks are the
+ /// responsibility of the application.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="uri">The Namespace URI, or the empty string if
+ /// none is available or Namespace processing is not
+ /// being performed.</param>
+ /// <param name="localName">The local name, or the empty string if
+ /// Namespace processing is not being performed.</param>
+ /// <param name="qName">The qualified name, or the empty string
+ /// if qualified names are not available.</param>
+ /// <param name="type">The attribute type as a string.</param>
+ /// <param name="value">The attribute value.</param>
+ /// <exception cref="IndexOutOfRangeException">When the
+ /// supplied index does not point to an attribute
+ /// in the list.</exception>
+ public virtual void SetAttribute(int index, string uri, string localName,
+ string qName, string type, string value)
+ {
+ if (index >= 0 && index < length)
+ {
+ data[index * 5] = uri;
+ data[index * 5 + 1] = localName;
+ data[index * 5 + 2] = qName;
+ data[index * 5 + 3] = type;
+ data[index * 5 + 4] = value;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ /// <summary>
+ /// Remove an attribute from the list.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <exception cref="IndexOutOfRangeException">When the supplied index does not point to an attribute in the list.</exception>
+ public virtual void RemoveAttribute(int index)
+ {
+ if (index >= 0 && index < length)
+ {
+ if (index < length - 1)
+ {
+ System.Array.Copy(data, (index + 1) * 5, data, index * 5,
+ (length - index - 1) * 5);
+ }
+ index = (length - 1) * 5;
+ data[index++] = null;
+ data[index++] = null;
+ data[index++] = null;
+ data[index++] = null;
+ data[index] = null;
+ length--;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ /// <summary>
+ /// Set the Namespace URI of a specific attribute.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="uri">The attribute's Namespace URI, or the empty
+ /// string for none.</param>
+ /// <exception cref="IndexOutOfRangeException">When the
+ /// supplied index does not point to an attribute
+ /// in the list.</exception>
+ public virtual void SetURI(int index, string uri)
+ {
+ if (index >= 0 && index < length)
+ {
+ data[index * 5] = uri;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ /// <summary>
+ /// Set the local name of a specific attribute.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="localName">The attribute's local name, or the empty
+ /// string for none.</param>
+ /// <exception cref="IndexOutOfRangeException">When the
+ /// supplied index does not point to an attribute
+ /// in the list.</exception>
+ public virtual void SetLocalName(int index, string localName)
+ {
+ if (index >= 0 && index < length)
+ {
+ data[index * 5 + 1] = localName;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ /// <summary>
+ /// Set the qualified name of a specific attribute.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="qName">The attribute's qualified name, or the empty
+ /// string for none.</param>
+ /// <exception cref="IndexOutOfRangeException">When the
+ /// supplied index does not point to an attribute
+ /// in the list.</exception>
+ public virtual void SetQName(int index, string qName)
+ {
+ if (index >= 0 && index < length)
+ {
+ data[index * 5 + 2] = qName;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ /// <summary>
+ /// Set the type of a specific attribute.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="type">The attribute's type.</param>
+ /// <exception cref="IndexOutOfRangeException">When the
+ /// supplied index does not point to an attribute
+ /// in the list.</exception>
+ public virtual void SetType(int index, string type)
+ {
+ if (index >= 0 && index < length)
+ {
+ data[index * 5 + 3] = type;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ /// <summary>
+ /// Set the value of a specific attribute.
+ /// </summary>
+ /// <param name="index">The index of the attribute (zero-based).</param>
+ /// <param name="value">The attribute's value.</param>
+ /// <exception cref="IndexOutOfRangeException">When the
+ /// supplied index does not point to an attribute
+ /// in the list.</exception>
+ public virtual void SetValue(int index, string value)
+ {
+ if (index >= 0 && index < length)
+ {
+ data[index * 5 + 4] = value;
+ }
+ else
+ {
+ BadIndex(index);
+ }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal methods.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Ensure the internal array's capacity.
+ /// </summary>
+ /// <param name="n">The minimum number of attributes that the array must be able to hold.</param>
+ private void EnsureCapacity(int n)
+ {
+ if (n <= 0)
+ {
+ return;
+ }
+ int max;
+ if (data == null || data.Length == 0)
+ {
+ max = 25;
+ }
+ else if (data.Length >= n * 5)
+ {
+ return;
+ }
+ else
+ {
+ max = data.Length;
+ }
+ while (max < n * 5)
+ {
+ max *= 2;
+ }
+
+ string[] newData = new string[max];
+ if (length > 0)
+ {
+ System.Array.Copy(data, 0, newData, 0, length * 5);
+ }
+ data = newData;
+ }
+
+ /// <summary>
+ /// Report a bad array index in a manipulator.
+ /// </summary>
+ /// <param name="index">The index to report.</param>
+ /// <exception cref="IndexOutOfRangeException">Always.</exception>
+ private void BadIndex(int index)
+ {
+ string msg =
+ "Attempt to modify attribute at illegal index: " + index;
+ throw new IndexOutOfRangeException(msg);
+ }
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////////
+
+ int length;
+ string[] data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Helpers/DefaultHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Helpers/DefaultHandler.cs b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/DefaultHandler.cs
new file mode 100644
index 0000000..d3523a4
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/DefaultHandler.cs
@@ -0,0 +1,389 @@
+// DefaultHandler.java - default implementation of the core handlers.
+// http://www.saxproject.org
+// Written by David Megginson
+// NO WARRANTY! This class is in the public domain.
+// $Id: DefaultHandler.java,v 1.9 2004/04/26 17:34:35 dmegginson Exp $
+
+namespace Sax.Helpers
+{
+ /// <summary>
+ /// Default base class for SAX2 event handlers.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class is available as a convenience base class for SAX2
+ /// applications: it provides default implementations for all of the
+ /// callbacks in the four core SAX2 handler classes:
+ /// <list type="number">
+ /// <item><description><see cref="IEntityResolver"/></description></item>
+ /// <item><description><see cref="IDTDHandler"/></description></item>
+ /// <item><description><see cref="IContentHandler"/></description></item>
+ /// <item><description><see cref="IErrorHandler"/></description></item>
+ /// </list>
+ /// <para/>
+ /// Application writers can extend this class when they need to
+ /// implement only part of an interface; parser writers can
+ /// instantiate this class to provide default handlers when the
+ /// application has not supplied its own.
+ /// <para/>
+ /// This class replaces the deprecated SAX1
+ /// Sax.HandlerBase class.
+ /// </remarks>
+ /// <since>SAX 2.0</since>
+ /// <author>David Megginson,</author>
+ /// <version>2.0.1 (sax2r2)</version>
+ /// <seealso cref="IEntityResolver"/>
+ /// <seealso cref="IDTDHandler"/>
+ /// <seealso cref="IContentHandler"/>
+ /// <seealso cref="IErrorHandler"/>
+ public class DefaultHandler : IEntityResolver, IDTDHandler, IContentHandler, IErrorHandler
+ {
+ ////////////////////////////////////////////////////////////////////
+ // Default implementation of the EntityResolver interface.
+ ////////////////////////////////////////////////////////////////////
+
+ /// <summary>
+ /// Resolve an external entity.
+ /// <para/>
+ /// Always return null, so that the parser will use the system
+ /// identifier provided in the XML document. This method implements
+ /// the SAX default behaviour: application writers can override it
+ /// in a subclass to do special translations such as catalog lookups
+ /// or URI redirection.
+ /// </summary>
+ /// <param name="publicId">The public identifer, or null if none is available.</param>
+ /// <param name="systemId">The system identifier provided in the XML document.</param>
+ /// <remarks>The new input source, or null to require the default behaviour.</remarks>
+ /// <exception cref="System.IO.IOException">If there is an error setting
+ /// up the new input source.</exception>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IEntityResolver.ResolveEntity(string, string)"/>
+ public virtual InputSource ResolveEntity(string publicId, string systemId)
+ {
+ return null;
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Default implementation of DTDHandler interface.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Receive notification of a notation declaration.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass if they wish to keep track of the notations
+ /// declared in a document.
+ /// </summary>
+ /// <param name="name">The notation name.</param>
+ /// <param name="publicId">The notation public identifier, or null if not
+ /// available.</param>
+ /// <param name="systemId">The notation system identifier.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IDTDHandler.NotationDecl(string, string, string)"/>
+ public virtual void NotationDecl(string name, string publicId, string systemId)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of an unparsed entity declaration.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to keep track of the unparsed entities
+ /// declared in a document.
+ /// </summary>
+ /// <param name="name">The entity name.</param>
+ /// <param name="publicId">The entity public identifier, or null if not available.</param>
+ /// <param name="systemId">The entity system identifier.</param>
+ /// <param name="notationName">The name of the associated notation.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IDTDHandler.UnparsedEntityDecl(string, string, string, string)"/>
+ public virtual void UnparsedEntityDecl(string name, string publicId,
+ string systemId, string notationName)
+ {
+ // no op
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Default implementation of ContentHandler interface.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Receive a Locator object for document events.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass if they wish to store the locator for use
+ /// with other document events.
+ /// </summary>
+ /// <param name="locator">A locator for all SAX document events.</param>
+ /// <seealso cref="IContentHandler.SetDocumentLocator(ILocator)"/>
+ /// <seealso cref="ILocator"/>
+ public virtual void SetDocumentLocator(ILocator locator)
+ {
+ // no op
+ }
+
+ /// <summary>
+ /// Receive notification of the beginning of the document.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions at the beginning
+ /// of a document (such as allocating the root node of a tree or
+ /// creating an output file).
+ /// </summary>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.StartDocument()"/>
+ public virtual void StartDocument()
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of the end of the document.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions at the end
+ /// of a document (such as finalising a tree or closing an output
+ /// file).
+ /// </summary>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.EndDocument()"/>
+ public virtual void EndDocument()
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of the start of a Namespace mapping.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions at the start of
+ /// each Namespace prefix scope (such as storing the prefix mapping).
+ /// </summary>
+ /// <param name="prefix">The Namespace prefix being declared.</param>
+ /// <param name="uri">The Namespace URI mapped to the prefix.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.StartPrefixMapping(string, string)"/>
+ public virtual void StartPrefixMapping(string prefix, string uri)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of the end of a Namespace mapping.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions at the end of
+ /// each prefix mapping.
+ /// </summary>
+ /// <param name="prefix">The Namespace prefix being declared.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.EndPrefixMapping(string)"/>
+ public virtual void EndPrefixMapping(string prefix)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of the start of an element.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions at the start of
+ /// each element (such as allocating a new tree node or writing
+ /// output to a file).
+ /// </summary>
+ /// <param name="prefix">The Namespace URI, or the empty string if the
+ /// element has no Namespace URI or if Namespace
+ /// processing is not being performed.</param>
+ /// <param name="localName">The local name (without prefix), or the
+ /// empty string if Namespace processing is not being
+ /// performed.</param>
+ /// <param name="qName">The qualified name (with prefix), or the
+ /// empty string if qualified names are not available.</param>
+ /// <param name="attributes">The attributes attached to the element. If
+ /// there are no attributes, it shall be an empty
+ /// <see cref="Attributes"/> object.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.StartElement(string, string, string, IAttributes)"/>
+ public virtual void StartElement(string uri, string localName,
+ string qName, IAttributes attributes)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of the end of an element.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions at the end of
+ /// each element (such as finalising a tree node or writing
+ /// output to a file).
+ /// </summary>
+ /// <param name="uri">The Namespace URI, or the empty string if the
+ /// element has no Namespace URI or if Namespace
+ /// processing is not being performed.</param>
+ /// <param name="localName">The local name (without prefix), or the
+ /// empty string if Namespace processing is not being
+ /// performed.</param>
+ /// <param name="qName">The qualified name (with prefix), or the
+ /// empty string if qualified names are not available.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.EndElement(string, string, string)"/>
+ public virtual void EndElement(string uri, string localName, string qName)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of character data inside an element.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method to take specific actions for each chunk of character data
+ /// (such as adding the data to a node or buffer, or printing it to
+ /// a file).
+ /// </summary>
+ /// <param name="ch">The characters.</param>
+ /// <param name="start">The start position in the character array.</param>
+ /// <param name="length">The number of characters to use from the character array.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.Characters(char[], int, int)"/>
+ public virtual void Characters(char[] ch, int start, int length)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of ignorable whitespace in element content.
+ /// <para/>
+ /// By default, do nothing. Application writers may override this
+ /// method to take specific actions for each chunk of ignorable
+ /// whitespace (such as adding data to a node or buffer, or printing
+ /// it to a file).
+ /// </summary>
+ /// <param name="ch">The whitespace characters.</param>
+ /// <param name="start">The start position in the character array.</param>
+ /// <param name="length">The number of characters to use from the character array.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.IgnorableWhitespace(char[], int, int)"/>
+ public virtual void IgnorableWhitespace(char[] ch, int start, int length)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of a processing instruction.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions for each
+ /// processing instruction, such as setting status variables or
+ /// invoking other methods.
+ /// </summary>
+ /// <param name="target">The processing instruction target.</param>
+ /// <param name="data">The processing instruction data, or null if
+ /// none is supplied.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.ProcessingInstruction(string, string)"/>
+ public virtual void ProcessingInstruction(string target, string data)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of a skipped entity.
+ /// <para/>By default, do nothing. Application writers may override this
+ /// method in a subclass to take specific actions for each
+ /// processing instruction, such as setting status variables or
+ /// invoking other methods.
+ /// </summary>
+ /// <param name="name">The name of the skipped entity.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IContentHandler.ProcessingInstruction(string, string)"/>
+ public virtual void SkippedEntity(string name)
+ {
+ // no op
+ }
+
+
+
+ ////////////////////////////////////////////////////////////////////
+ // Default implementation of the ErrorHandler interface.
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Receive notification of a parser warning.
+ /// <para/>
+ /// The default implementation does nothing. Application writers
+ /// may override this method in a subclass to take specific actions
+ /// for each warning, such as inserting the message in a log file or
+ /// printing it to the console.
+ /// </summary>
+ /// <param name="e">The warning information encoded as an exception.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IErrorHandler.Warning(SAXParseException)"/>
+ /// <seealso cref="SAXParseException"/>
+ public virtual void Warning(SAXParseException e)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Receive notification of a recoverable parser error.
+ /// <para/>The default implementation does nothing. Application writers
+ /// may override this method in a subclass to take specific actions
+ /// for each error, such as inserting the message in a log file or
+ /// printing it to the console.
+ /// </summary>
+ /// <param name="e">The warning information encoded as an exception.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IErrorHandler.Warning(SAXParseException)"/>
+ /// <seealso cref="SAXParseException"/>
+ public virtual void Error(SAXParseException e)
+ {
+ // no op
+ }
+
+
+ /// <summary>
+ /// Report a fatal XML parsing error.
+ /// <para/>
+ /// The default implementation throws a <see cref="SAXParseException"/>.
+ /// Application writers may override this method in a subclass if
+ /// they need to take specific actions for each fatal error (such as
+ /// collecting all of the errors into a single report): in any case,
+ /// the application must stop all regular processing when this
+ /// method is invoked, since the document is no longer reliable, and
+ /// the parser may no longer report parsing events.
+ /// </summary>
+ /// <param name="e">The error information encoded as an exception.</param>
+ /// <exception cref="SAXException">Any SAX exception, possibly
+ /// wrapping another exception.</exception>
+ /// <seealso cref="IErrorHandler.FatalError(SAXParseException)"/>
+ /// <seealso cref="SAXParseException"/>
+ public virtual void FatalError(SAXParseException e)
+ {
+ throw e;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/198e5868/src/Lucene.Net.Benchmark/Support/Sax/Helpers/LocatorImpl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/Support/Sax/Helpers/LocatorImpl.cs b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/LocatorImpl.cs
new file mode 100644
index 0000000..8356240
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/Support/Sax/Helpers/LocatorImpl.cs
@@ -0,0 +1,131 @@
+// SAX default implementation for Locator.
+// http://www.saxproject.org
+// No warranty; no copyright -- use this as you will.
+// $Id: LocatorImpl.java,v 1.6 2002/01/30 20:52:27 dbrownell Exp $
+
+namespace Sax.Helpers
+{
+ /// <summary>
+ /// Provide an optional convenience implementation of <see cref="ILocator"/>.
+ /// </summary>
+ /// <remarks>
+ /// <em>This module, both source code and documentation, is in the
+ /// Public Domain, and comes with<strong> NO WARRANTY</strong>.</em>
+ /// See<a href='http://www.saxproject.org'>http://www.saxproject.org</a>
+ /// for further information.
+ /// <para/>
+ /// This class is available mainly for application writers, who
+ /// can use it to make a persistent snapshot of a locator at any
+ /// point during a document parse:
+ /// <code>
+ /// ILocator locator;
+ /// ILocator startloc;
+ ///
+ /// public void SetLocator(ILocator locator)
+ /// {
+ /// // note the locator
+ /// this.locator = locator;
+ /// }
+ ///
+ /// public void StartDocument()
+ /// {
+ /// // save the location of the start of the document
+ /// // for future use.
+ /// ILocator startloc = new Locator(locator);
+ /// }
+ /// </code>
+ /// <para/>
+ /// Normally, parser writers will not use this class, since it
+ /// is more efficient to provide location information only when
+ /// requested, rather than constantly updating a <see cref="ILocator"/> object.
+ /// </remarks>
+ public class Locator : ILocator
+ {
+ /// <summary>
+ /// Zero-argument constructor.
+ /// <para/>This will not normally be useful, since the main purpose
+ /// of this class is to make a snapshot of an existing <see cref="ILocator"/>.
+ /// <summary>
+ public Locator()
+ {
+ }
+
+ /// <summary>
+ /// Copy constructor.
+ /// <para/>
+ /// Create a persistent copy of the current state of a locator.
+ /// When the original locator changes, this copy will still keep
+ /// the original values (and it can be used outside the scope of
+ /// DocumentHandler methods).
+ /// <summary>
+ /// <param name="locator">The locator to copy.</param>
+ public Locator(ILocator locator)
+ {
+ publicId = locator.PublicId;
+ systemId = locator.SystemId;
+ lineNumber = locator.LineNumber;
+ columnNumber = locator.ColumnNumber;
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Implementation of org.xml.sax.Locator
+ ////////////////////////////////////////////////////////////////////
+
+
+ /// <summary>
+ /// Gets the public identifier as a string, or null if none
+ /// is available.
+ /// </summary>
+ /// <seealso cref="ILocator.PublicId"/>
+ public string PublicId
+ {
+ get { return publicId; }
+ set { publicId = value; }
+ }
+
+
+ /// <summary>
+ /// Gets the system identifier as a string, or null if none
+ /// is available.
+ /// </summary>
+ /// <seealso cref="ILocator.SystemId"/>
+ public string SystemId
+ {
+ get { return systemId; }
+ set { systemId = value; }
+ }
+
+
+ /// <summary>
+ /// Gets the saved line number (1-based).
+ /// Returns the line number as an integer, or -1 if none is available.
+ /// </summary>
+ /// <seealso cref="ILocator.LineNumber"/>
+ public int LineNumber
+ {
+ get { return lineNumber; }
+ set { lineNumber = value; }
+ }
+
+
+ /// <summary>
+ /// Gets the saved column number (1-based).
+ /// Returns the column number as an integer, or -1 if none is available.
+ /// </summary>
+ /// <seealso cref="ILocator.ColumnNumber"/>
+ public int ColumnNumber
+ {
+ get { return columnNumber; }
+ set { columnNumber = value; }
+ }
+
+ ////////////////////////////////////////////////////////////////////
+ // Internal state.
+ ////////////////////////////////////////////////////////////////////
+
+ private string publicId;
+ private string systemId;
+ private int lineNumber;
+ private int columnNumber;
+ }
+}
[30/33] lucenenet git commit: Lucene.Net.Spatial: Removed unnecessary
dependencies on GeoAPI and NetTopologySuite
Posted by ni...@apache.org.
Lucene.Net.Spatial: Removed unnecessary dependencies on GeoAPI and NetTopologySuite
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/14c1760f
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/14c1760f
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/14c1760f
Branch: refs/heads/master
Commit: 14c1760f5310781822f8542c54b67e4a6798ffd9
Parents: ea879c6
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Sun Aug 6 13:51:57 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Sun Aug 6 13:51:57 2017 +0700
----------------------------------------------------------------------
src/Lucene.Net.Spatial/project.json | 4 +---
src/Lucene.Net.Tests.Spatial/project.json | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/14c1760f/src/Lucene.Net.Spatial/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Spatial/project.json b/src/Lucene.Net.Spatial/project.json
index ead4ac8..bcad265 100644
--- a/src/Lucene.Net.Spatial/project.json
+++ b/src/Lucene.Net.Spatial/project.json
@@ -19,13 +19,11 @@
},
"dependencies": {
"Lucene.Net.Queries": "4.8.0",
- "GeoAPI": "1.7.4",
- "NetTopologySuite": "1.14",
"Spatial4n.Core": "0.4.1-beta00003"
},
"frameworks": {
"netstandard1.5": {
- "imports": [ "dnxcore50", "portable-net403+sl5+win8+wp8+wpa81" ],
+ "imports": "dnxcore50",
"buildOptions": {
"debugType": "portable",
"define": [ "NETSTANDARD" ]
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/14c1760f/src/Lucene.Net.Tests.Spatial/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Spatial/project.json b/src/Lucene.Net.Tests.Spatial/project.json
index a29f7f5..934f704 100644
--- a/src/Lucene.Net.Tests.Spatial/project.json
+++ b/src/Lucene.Net.Tests.Spatial/project.json
@@ -33,7 +33,7 @@
"testRunner": "nunit-teamcity",
"frameworks": {
"netcoreapp1.0": {
- "imports": [ "dnxcore50", "portable-net403+sl5+win8+wp8+wpa81" ],
+ "imports": "dnxcore50",
"buildOptions": {
"debugType": "portable",
"define": [ "NETSTANDARD" ]
[03/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TestHtmlParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TestHtmlParser.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TestHtmlParser.cs
new file mode 100644
index 0000000..ce9f2f8
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TestHtmlParser.cs
@@ -0,0 +1,164 @@
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using static Lucene.Net.Benchmarks.ByTask.Feeds.DemoHTMLParser;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class TestHtmlParser : LuceneTestCase
+ {
+ [Test]
+ public void TestUnicode()
+ {
+ String text = "<html><body>汉语</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("汉语", parser.Body);
+ }
+
+ [Test]
+ public void TestEntities()
+ {
+ String text = "<html><body>汉语¥</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("汉语¥", parser.Body);
+ }
+
+ [Test]
+ public void TestComments()
+ {
+ String text = "<html><body>foo<!-- bar --><! baz --></body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("foo", parser.Body);
+ }
+
+ [Test]
+ public void TestScript()
+ {
+ String text = "<html><body><script type=\"text/javascript\">" +
+ "document.write(\"test\")</script>foo</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("foo", parser.Body);
+ }
+
+ [Test]
+ public void TestStyle()
+ {
+ String text = "<html><head><style type=\"text/css\">" +
+ "body{background-color:blue;}</style>" +
+ "</head><body>foo</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("foo", parser.Body);
+ }
+
+ [Test]
+ public void TestDoctype()
+ {
+ String text = "<!DOCTYPE HTML PUBLIC " +
+ "\"-//W3C//DTD HTML 4.01 Transitional//EN\"" +
+ "\"http://www.w3.org/TR/html4/loose.dtd\">" +
+ "<html><body>foo</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("foo", parser.Body);
+ }
+
+ [Test]
+ public void TestMeta()
+ {
+ String text = "<html><head>" +
+ "<meta name=\"a\" content=\"1\" />" +
+ "<meta name=\"b\" content=\"2\" />" +
+ "<meta name=\"keywords\" content=\"this is a test\" />" +
+ "<meta http-equiv=\"Content-Type\" content=\"text/html;charset=UTF-8\" />" +
+ "</head><body>foobar</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ IDictionary<string, string> tags = parser.MetaTags;
+ assertEquals(4, tags.size());
+ assertEquals("1", tags["a"]);
+ assertEquals("2", tags["b"]);
+ assertEquals("this is a test", tags["keywords"]);
+ assertEquals("text/html;charset=UTF-8", tags["content-type"]);
+ }
+
+ [Test]
+ public void TestTitle()
+ {
+ String text = "<html><head><TITLE>foo</TITLE><head><body>bar</body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("foo", parser.Title);
+ }
+
+ // LUCENE-2246
+ [Test]
+ public void TestTurkish()
+ {
+ using (var context = new CultureContext("tr-TR"))
+ {
+ String text = "<html><HEAD><TITLE>ııı</TITLE></head><body>" +
+ "<IMG SRC=\"../images/head.jpg\" WIDTH=570 HEIGHT=47 BORDER=0 ALT=\"ş\">" +
+ "<a title=\"(ııı)\"></body></html>";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("ııı", parser.Title);
+ assertEquals("[ş]", parser.Body);
+ }
+ }
+
+ [Test]
+ public void TestSampleTRECDoc()
+ {
+ String text = "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-000 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-000 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("TEST-000 title", parser.Title);
+ assertEquals("TEST-000 text", parser.Body.Trim());
+ }
+
+ [Test]
+ public void TestNoHTML()
+ {
+ String text = "hallo";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("", parser.Title);
+ assertEquals("hallo", parser.Body);
+ }
+
+ [Test]
+ public void Testivalid()
+ {
+ String text = "<title>foo</title>bar";
+ Parser parser = new Parser(new StringReader(text));
+ assertEquals("foo", parser.Title);
+ assertEquals("bar", parser.Body);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TrecContentSourceTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TrecContentSourceTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TrecContentSourceTest.cs
new file mode 100644
index 0000000..d83bb5a
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/TrecContentSourceTest.cs
@@ -0,0 +1,431 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using static Lucene.Net.Benchmarks.ByTask.Feeds.TrecDocParser;
+
+namespace Lucene.Net.Benchmarks.ByTask.Feeds
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class TrecContentSourceTest : LuceneTestCase
+ {
+ /** A TrecDocMaker which works on a String and not files. */
+ private class StringableTrecSource : TrecContentSource
+ {
+
+
+ private String docs = null;
+
+ public StringableTrecSource(String docs, bool forever)
+ {
+ this.docs = docs;
+ this.m_forever = forever;
+ }
+
+ internal override void OpenNextFile()
+ {
+ if (reader != null)
+ {
+ if (!m_forever)
+ {
+ throw new NoMoreDataException();
+ }
+ ++iteration;
+ }
+
+ reader = new StringReader(docs);
+ }
+
+ public override void SetConfig(Config config)
+ {
+ htmlParser = new DemoHTMLParser();
+ }
+ }
+
+ private void assertDocData(DocData dd, String expName, String expTitle,
+ String expBody, DateTime? expDate)
+ {
+ assertNotNull(dd);
+ assertEquals(expName, dd.Name);
+ assertEquals(expTitle, dd.Title);
+ assertTrue(dd.Body.IndexOf(expBody) != -1);
+ DateTime? date = dd.Date != null ? DateTools.StringToDate(dd.Date) : (DateTime?)null;
+ assertEquals(expDate, date);
+ }
+
+ private void assertNoMoreDataException(StringableTrecSource stdm)
+ {
+ bool thrown = false;
+ try
+ {
+ stdm.GetNextDocData(null);
+ }
+#pragma warning disable 168
+ catch (NoMoreDataException e)
+#pragma warning restore 168
+ {
+ thrown = true;
+ }
+ assertTrue("Expecting NoMoreDataException", thrown);
+ }
+
+ [Test]
+ public void TestOneDocument()
+ {
+ String docs = "<DOC>\r\n" +
+ "<DOCNO>TEST-000</DOCNO>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Date: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-000 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-000 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>";
+ StringableTrecSource source = new StringableTrecSource(docs, false);
+ source.SetConfig(null);
+
+ DocData dd = source.GetNextDocData(new DocData());
+ assertDocData(dd, "TEST-000_0", "TEST-000 title", "TEST-000 text", source
+ .ParseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+
+
+ assertNoMoreDataException(source);
+ }
+
+ [Test]
+ public void TestTwoDocuments()
+ {
+ String docs = "<DOC>\r\n" +
+ "<DOCNO>TEST-000</DOCNO>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Date: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-000 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-000 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>\r\n" +
+ "<DOC>\r\n" +
+ "<DOCNO>TEST-001</DOCNO>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Date: Sun, 11 Jan 2009 08:01:00 GMT\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2008 08:01:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-001 title\r\n" +
+ "</title>\r\n" +
+ "<meta name=\"date\" content=\"Tue, 09 Dec 2003 22:39:08 GMT\">" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-001 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>";
+ StringableTrecSource source = new StringableTrecSource(docs, false);
+ source.SetConfig(null);
+
+ DocData dd = source.GetNextDocData(new DocData());
+ assertDocData(dd, "TEST-000_0", "TEST-000 title", "TEST-000 text", source
+ .ParseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+
+ dd = source.GetNextDocData(dd);
+ assertDocData(dd, "TEST-001_0", "TEST-001 title", "TEST-001 text", source
+ .ParseDate("Tue, 09 Dec 2003 22:39:08 GMT"));
+
+
+ assertNoMoreDataException(source);
+ }
+
+ // If a Date: attribute is missing, make sure the document is not skipped, but
+ // rather that null Data is assigned.
+ [Test]
+ public void TestMissingDate()
+ {
+ String docs = "<DOC>\r\n" +
+ "<DOCNO>TEST-000</DOCNO>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-000 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-000 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>\r\n" +
+ "<DOC>\r\n" +
+ "<DOCNO>TEST-001</DOCNO>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Date: Sun, 11 Jan 2009 08:01:00 GMT\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2009 08:01:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-001 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-001 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>";
+ StringableTrecSource source = new StringableTrecSource(docs, false);
+ source.SetConfig(null);
+
+ DocData dd = source.GetNextDocData(new DocData());
+ assertDocData(dd, "TEST-000_0", "TEST-000 title", "TEST-000 text", null);
+
+ dd = source.GetNextDocData(dd);
+ assertDocData(dd, "TEST-001_0", "TEST-001 title", "TEST-001 text", source
+ .ParseDate("Sun, 11 Jan 2009 08:01:00 GMT"));
+
+
+ assertNoMoreDataException(source);
+ }
+
+ // When a 'bad date' is input (unparsable date), make sure the DocData date is
+ // assigned null.
+ [Test]
+ public void TestBadDate()
+ {
+ String docs = "<DOC>\r\n" +
+ "<DOCNO>TEST-000</DOCNO>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Date: Bad Date\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-000 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-000 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>";
+ StringableTrecSource source = new StringableTrecSource(docs, false);
+ source.SetConfig(null);
+
+ DocData dd = source.GetNextDocData(new DocData());
+ assertDocData(dd, "TEST-000_0", "TEST-000 title", "TEST-000 text", null);
+
+
+ assertNoMoreDataException(source);
+ }
+
+ [Test]
+ public void TestForever()
+ {
+ String docs = "<DOC>\r\n" +
+ "<DOCNO>TEST-000</DOCNO>\r\n" +
+ //"<docno>TEST-000</docno>\r\n" +
+ "<DOCHDR>\r\n" +
+ "http://lucene.apache.org.trecdocmaker.test\r\n" +
+ "HTTP/1.1 200 OK\r\n" +
+ "Date: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Server: Apache/1.3.27 (Unix)\r\n" +
+ "Last-Modified: Sun, 11 Jan 2009 08:00:00 GMT\r\n" +
+ "Content-Length: 614\r\n" +
+ "Connection: close\r\n" +
+ "Content-Type: text/html\r\n" +
+ "</DOCHDR>\r\n" +
+ "<html>\r\n" +
+ "\r\n" +
+ "<head>\r\n" +
+ "<title>\r\n" +
+ "TEST-000 title\r\n" +
+ "</title>\r\n" +
+ "</head>\r\n" +
+ "\r\n" +
+ "<body>\r\n" +
+ "TEST-000 text\r\n" +
+ "\r\n" +
+ "</body>\r\n" +
+ "\r\n" +
+ "</DOC>";
+ StringableTrecSource source = new StringableTrecSource(docs, true);
+ source.SetConfig(null);
+
+ DocData dd = source.GetNextDocData(new DocData());
+ assertDocData(dd, "TEST-000_0", "TEST-000 title", "TEST-000 text", source
+ .ParseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+
+ // same document, but the second iteration changes the name.
+ dd = source.GetNextDocData(dd);
+ assertDocData(dd, "TEST-000_1", "TEST-000 title", "TEST-000 text", source
+ .ParseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+ source.Dispose();
+
+ // Don't test that NoMoreDataException is thrown, since the forever flag is
+ // turned on.
+ }
+
+ /**
+ * Open a trec content source over a directory with files of all trec path types and all
+ * supported formats - bzip, gzip, txt.
+ */
+ [Test]
+ public void TestTrecFeedDirAllTypes()
+ {
+ DirectoryInfo dataDir = CreateTempDir("trecFeedAllTypes");
+ using (var stream = GetDataFile("trecdocs.zip"))
+ TestUtil.Unzip(stream, dataDir);
+ TrecContentSource tcs = new TrecContentSource();
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["print.props"] = "false";
+ props["content.source.verbose"] = "false";
+ props["content.source.excludeIteration"] = "true";
+ props["doc.maker.forever"] = "false";
+ props["docs.dir"] = dataDir.FullName.Replace('\\', '/');
+ props["trec.doc.parser"] = typeof(TrecParserByPath).AssemblyQualifiedName;
+ props["content.source.forever"] = "false";
+ tcs.SetConfig(new Config(props));
+ tcs.ResetInputs();
+ DocData dd = new DocData();
+ int n = 0;
+ bool gotExpectedException = false;
+ // LUCENENET specific - skip our UNKNOWN element.
+ var pathTypes = ((ParsePathType[])Enum.GetValues(typeof(ParsePathType))).Where(x => x != ParsePathType.UNKNOWN).ToArray();
+ HashSet<ParsePathType> unseenTypes = new HashSet<ParsePathType>(pathTypes);
+ try
+ {
+ while (n < 100)
+ { // arbiterary limit to prevent looping forever in case of test failure
+ dd = tcs.GetNextDocData(dd);
+ ++n;
+ assertNotNull("doc data " + n + " should not be null!", dd);
+ unseenTypes.Remove(tcs.currPathType);
+ switch (tcs.currPathType)
+ {
+ case ParsePathType.GOV2:
+ assertDocData(dd, "TEST-000", "TEST-000 title", "TEST-000 text", tcs.ParseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+ break;
+ case ParsePathType.FBIS:
+ assertDocData(dd, "TEST-001", "TEST-001 Title", "TEST-001 text", tcs.ParseDate("1 January 1991"));
+ break;
+ case ParsePathType.FR94:
+ // no title extraction in this source for now
+ assertDocData(dd, "TEST-002", null, "DEPARTMENT OF SOMETHING", tcs.ParseDate("February 3, 1994"));
+ break;
+ case ParsePathType.FT:
+ assertDocData(dd, "TEST-003", "Test-003 title", "Some pub text", tcs.ParseDate("980424"));
+ break;
+ case ParsePathType.LATIMES:
+ assertDocData(dd, "TEST-004", "Test-004 Title", "Some paragraph", tcs.ParseDate("January 17, 1997, Sunday"));
+ break;
+ default:
+ assertTrue("Should never get here!", false);
+ break;
+ }
+ }
+ }
+#pragma warning disable 168
+ catch (NoMoreDataException e)
+#pragma warning restore 168
+ {
+ gotExpectedException = true;
+ }
+ assertTrue("Should have gotten NoMoreDataException!", gotExpectedException);
+ assertEquals("Wrong number of documents created by source!", 5, n);
+ assertTrue("Did not see all types!", unseenTypes.Count == 0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/trecdocs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/trecdocs.zip b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/trecdocs.zip
new file mode 100644
index 0000000..f12dbca
Binary files /dev/null and b/src/Lucene.Net.Tests.Benchmark/ByTask/Feeds/trecdocs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/AddIndexesTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/AddIndexesTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/AddIndexesTaskTest.cs
new file mode 100644
index 0000000..5fea5a5
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/AddIndexesTaskTest.cs
@@ -0,0 +1,153 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Store;
+using NUnit.Framework;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link AddIndexesTask}.
+ /// </summary>
+ public class AddIndexesTaskTest : BenchmarkTestCase
+ {
+ private static DirectoryInfo testDir, inputDir;
+
+ public override void BeforeClass()
+ {
+ base.BeforeClass();
+ testDir = CreateTempDir("addIndexesTask");
+
+ // create a dummy index under inputDir
+ inputDir = new DirectoryInfo(Path.Combine(testDir.FullName, "input"));
+ Store.Directory tmpDir = NewFSDirectory(inputDir);
+ try
+ {
+ IndexWriter writer = new IndexWriter(tmpDir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+ for (int i = 0; i < 10; i++)
+ {
+ writer.AddDocument(new Document());
+ }
+ writer.Dispose();
+ }
+ finally
+ {
+ tmpDir.Dispose();
+ }
+ }
+
+
+ private PerfRunData createPerfRunData()
+ {
+ IDictionary<string, string> props = new Dictionary<string, string>();
+ props["writer.version"] = TEST_VERSION_CURRENT.ToString();
+ props["print.props"] = "false"; // don't print anything
+ props["directory"] = "RAMDirectory";
+ props[AddIndexesTask.ADDINDEXES_INPUT_DIR] = inputDir.FullName;
+ Config config = new Config(props);
+ return new PerfRunData(config);
+ }
+
+ private void assertIndex(PerfRunData runData)
+ {
+ Store.Directory taskDir = runData.Directory;
+ assertSame(typeof(RAMDirectory), taskDir.GetType());
+ IndexReader r = DirectoryReader.Open(taskDir);
+ try
+ {
+ assertEquals(10, r.NumDocs);
+ }
+ finally
+ {
+ r.Dispose();
+ }
+ }
+
+ [Test]
+ public void TestAddIndexesDefault()
+ {
+ PerfRunData runData = createPerfRunData();
+ // create the target index first
+ new CreateIndexTask(runData).DoLogic();
+
+ AddIndexesTask task = new AddIndexesTask(runData);
+ task.Setup();
+
+ // add the input index
+ task.DoLogic();
+
+ // close the index
+ new CloseIndexTask(runData).DoLogic();
+
+
+ assertIndex(runData);
+
+ runData.Dispose();
+ }
+
+ [Test]
+ public void TestAddIndexesDir()
+ {
+ PerfRunData runData = createPerfRunData();
+ // create the target index first
+ new CreateIndexTask(runData).DoLogic();
+
+ AddIndexesTask task = new AddIndexesTask(runData);
+ task.Setup();
+
+ // add the input index
+ task.SetParams("true");
+ task.DoLogic();
+
+ // close the index
+ new CloseIndexTask(runData).DoLogic();
+
+
+ assertIndex(runData);
+
+ runData.Dispose();
+ }
+
+ [Test]
+ public void TestAddIndexesReader()
+ {
+ PerfRunData runData = createPerfRunData();
+ // create the target index first
+ new CreateIndexTask(runData).DoLogic();
+
+ AddIndexesTask task = new AddIndexesTask(runData);
+ task.Setup();
+
+ // add the input index
+ task.SetParams("false");
+ task.DoLogic();
+
+ // close the index
+ new CloseIndexTask(runData).DoLogic();
+
+
+ assertIndex(runData);
+
+ runData.Dispose();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltPackageTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltPackageTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltPackageTaskTest.cs
new file mode 100644
index 0000000..c8f4c79
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltPackageTaskTest.cs
@@ -0,0 +1,68 @@
+using NUnit.Framework;
+using System;
+using System.Reflection;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks.Alt
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests that tasks in alternate packages are found.
+ /// </summary>
+ public class AltPackageTaskTest : BenchmarkTestCase
+ {
+ /** Benchmark should fail loading the algorithm when alt is not specified */
+ [Test]
+ public void TestWithoutAlt()
+ {
+ try
+ {
+ execBenchmark(altAlg(false));
+ assertFalse("Should have failed to run the algorithm", true);
+ }
+#pragma warning disable 168
+ catch (Exception e)
+#pragma warning restore 168
+ {
+ // expected exception, do nothing
+ }
+ }
+
+ /** Benchmark should be able to load the algorithm when alt is specified */
+ [Test]
+ public void TestWithAlt()
+ {
+ Benchmark bm = execBenchmark(altAlg(true));
+ assertNotNull(bm);
+ assertNotNull(bm.RunData.Points);
+ }
+
+ private String[] altAlg(bool allowAlt)
+ {
+ String altTask = "{ AltTest }";
+ if (allowAlt)
+ {
+ return new String[] {
+ "alt.tasks.packages = " +this.GetType().GetTypeInfo().Assembly.GetName().Name,
+ altTask
+ };
+ }
+ return new String[] { altTask };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltTestTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltTestTask.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltTestTask.cs
new file mode 100644
index 0000000..7e13bb9
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/Alt/AltTestTask.cs
@@ -0,0 +1,35 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks.Alt
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// {@link PerfTask} which does nothing, but is in a different package
+ /// </summary>
+ public class AltTestTask : PerfTask
+ {
+ public AltTestTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CommitIndexTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CommitIndexTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CommitIndexTaskTest.cs
new file mode 100644
index 0000000..450916d
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CommitIndexTaskTest.cs
@@ -0,0 +1,63 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Index;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link CreateIndexTask}.
+ /// </summary>
+ public class CommitIndexTaskTest : BenchmarkTestCase
+ {
+ private PerfRunData createPerfRunData()
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["writer.version"] = TEST_VERSION_CURRENT.ToString();
+ props["print.props"] = "false"; // don't print anything
+ props["directory"] = "RAMDirectory";
+ Config config = new Config(props);
+ return new PerfRunData(config);
+ }
+
+ [Test]
+ public void TestNoParams()
+ {
+ PerfRunData runData = createPerfRunData();
+ new CreateIndexTask(runData).DoLogic();
+ new CommitIndexTask(runData).DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ }
+
+ [Test]
+ public void TestCommitData()
+ {
+ PerfRunData runData = createPerfRunData();
+ new CreateIndexTask(runData).DoLogic();
+ CommitIndexTask task = new CommitIndexTask(runData);
+ task.SetParams("params");
+ task.DoLogic();
+ SegmentInfos infos = new SegmentInfos();
+ infos.Read(runData.Directory);
+ assertEquals("params", infos.UserData[OpenReaderTask.USER_DATA]);
+ new CloseIndexTask(runData).DoLogic();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingHighlighterTestTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingHighlighterTestTask.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingHighlighterTestTask.cs
new file mode 100644
index 0000000..678339f
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingHighlighterTestTask.cs
@@ -0,0 +1,85 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Highlight;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test Search task which counts number of searches.
+ /// </summary>
+ public class CountingHighlighterTestTask : SearchTravRetHighlightTask
+ {
+ public static int numHighlightedResults = 0;
+ public static int numDocsRetrieved = 0;
+
+ public CountingHighlighterTestTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ protected override Document RetrieveDoc(IndexReader ir, int id)
+ {
+ Document document = ir.Document(id);
+ if (document != null)
+ {
+ numDocsRetrieved++;
+ }
+ return document;
+ }
+
+ private class BenchmarkHighlighterAnonymousHelper : BenchmarkHighlighter
+ {
+ private readonly CountingHighlighterTestTask outerInstance;
+ private readonly Highlighter highlighter;
+ public BenchmarkHighlighterAnonymousHelper(CountingHighlighterTestTask outerInstance, Highlighter highlighter)
+ {
+ this.outerInstance = outerInstance;
+ this.highlighter = highlighter;
+ }
+ public override int DoHighlight(IndexReader reader, int doc, string field, Document document, Analyzer analyzer, string text)
+ {
+ TokenStream ts = TokenSources.GetAnyTokenStream(reader, doc, field, document, analyzer);
+ TextFragment[]
+ frag = highlighter.GetBestTextFragments(ts, text, outerInstance.m_mergeContiguous, outerInstance.m_maxFrags);
+ numHighlightedResults += frag != null ? frag.Length : 0;
+ return frag != null ? frag.Length : 0;
+ }
+ }
+
+ protected override BenchmarkHighlighter GetBenchmarkHighlighter(Query q)
+ {
+ m_highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
+ return new BenchmarkHighlighterAnonymousHelper(this, m_highlighter);
+ // return new BenchmarkHighlighter() {
+ // @Override
+ // public int doHighlight(IndexReader reader, int doc, String field, Document document, Analyzer analyzer, String text)
+ // {
+ // TokenStream ts = TokenSources.GetAnyTokenStream(reader, doc, field, document, analyzer);
+ // TextFragment []
+ // frag = highlighter.GetBestTextFragments(ts, text, mergeContiguous, maxFrags);
+ // numHighlightedResults += frag != null ? frag.Length : 0;
+ // return frag != null ? frag.Length : 0;
+ // }
+ //};
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingSearchTestTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingSearchTestTask.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingSearchTestTask.cs
new file mode 100644
index 0000000..10c7628
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CountingSearchTestTask.cs
@@ -0,0 +1,65 @@
+using Lucene.Net.Support;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test Search task which counts number of searches.
+ /// </summary>
+ public class CountingSearchTestTask : SearchTask
+ {
+ public static int numSearches = 0;
+ public static long startMillis;
+ public static long lastMillis;
+ public static long prevLastMillis;
+
+ private static object syncLock = new object();
+
+ public CountingSearchTestTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ int res = base.DoLogic();
+ IncrNumSearches();
+ return res;
+ }
+
+ private static void IncrNumSearches()
+ {
+ lock (syncLock)
+ {
+ prevLastMillis = lastMillis;
+ lastMillis = Time.CurrentTimeMilliseconds();
+ if (0 == numSearches)
+ {
+ startMillis = prevLastMillis = lastMillis;
+ }
+ numSearches++;
+ }
+ }
+
+ public long GetElapsedMillis()
+ {
+ return lastMillis - startMillis;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CreateIndexTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CreateIndexTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CreateIndexTaskTest.cs
new file mode 100644
index 0000000..29cbaf7
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/CreateIndexTaskTest.cs
@@ -0,0 +1,129 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using Lucene.Net.Support.IO;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link CreateIndexTask}.
+ /// </summary>
+ public class CreateIndexTaskTest : BenchmarkTestCase
+ {
+ private PerfRunData createPerfRunData(String infoStreamValue)
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ // :Post-Release-Update-Version.LUCENE_XY:
+#pragma warning disable 612, 618
+ props["writer.version"] = LuceneVersion.LUCENE_47.ToString();
+#pragma warning restore 612, 618
+ props["print.props"] = "false"; // don't print anything
+ props["directory"] = "RAMDirectory";
+ if (infoStreamValue != null)
+ {
+ props["writer.info.stream"] = infoStreamValue;
+ }
+ Config config = new Config(props);
+ return new PerfRunData(config);
+ }
+
+ [Test]
+ public void TestInfoStream_SystemOutErr()
+ {
+
+ TextWriter curOut = SystemConsole.Out;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ SystemConsole.Out = new StreamWriter(baos, Encoding.GetEncoding(0));
+ try
+ {
+ PerfRunData runData = createPerfRunData("SystemOut");
+ CreateIndexTask cit = new CreateIndexTask(runData);
+ cit.DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ assertTrue(baos.Length > 0);
+ }
+ finally
+ {
+ SystemConsole.Out = curOut;
+ }
+
+ TextWriter curErr = SystemConsole.Error;
+ baos = new ByteArrayOutputStream();
+ SystemConsole.Error = new StreamWriter(baos, Encoding.GetEncoding(0));
+ try
+ {
+ PerfRunData runData = createPerfRunData("SystemErr");
+ CreateIndexTask cit = new CreateIndexTask(runData);
+ cit.DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ assertTrue(baos.Length > 0);
+ }
+ finally
+ {
+ SystemConsole.Error = curErr;
+ }
+
+ }
+
+ [Test]
+ public void TestInfoStream_File()
+ {
+
+ FileInfo outFile = new FileInfo(Path.Combine(getWorkDir().FullName, "infoStreamTest"));
+ PerfRunData runData = createPerfRunData(outFile.FullName);
+ new CreateIndexTask(runData).DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ assertTrue(new FileInfo(outFile.FullName).Length > 0);
+ }
+
+ [Test]
+ public void TestNoMergePolicy()
+ {
+ PerfRunData runData = createPerfRunData(null);
+ runData.Config.Set("merge.policy", typeof(NoMergePolicy).AssemblyQualifiedName);
+ new CreateIndexTask(runData).DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ }
+
+ [Test]
+ public void TestNoMergeScheduler()
+ {
+ PerfRunData runData = createPerfRunData(null);
+ runData.Config.Set("merge.scheduler", typeof(NoMergeScheduler).AssemblyQualifiedName);
+ new CreateIndexTask(runData).DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ }
+
+ [Test]
+ public void TestNoDeletionPolicy()
+ {
+ PerfRunData runData = createPerfRunData(null);
+ runData.Config.Set("deletion.policy", typeof(NoDeletionPolicy).AssemblyQualifiedName);
+ new CreateIndexTask(runData).DoLogic();
+ new CloseIndexTask(runData).DoLogic();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/PerfTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/PerfTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/PerfTaskTest.cs
new file mode 100644
index 0000000..572e109
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/PerfTaskTest.cs
@@ -0,0 +1,81 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using NUnit.Framework;
+using System.Collections.Generic;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of the abstract {@link PerfTask}.
+ /// </summary>
+ public class PerfTaskTest : BenchmarkTestCase
+ {
+ private sealed class MyPerfTask : PerfTask
+ {
+
+ public MyPerfTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ return 0;
+ }
+
+ public int getLogStep() { return m_logStep; }
+ }
+
+ private PerfRunData createPerfRunData(bool setLogStep, int logStepVal,
+ bool setTaskLogStep, int taskLogStepVal)
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ if (setLogStep)
+ {
+ props["log.step"] = logStepVal.ToString(CultureInfo.InvariantCulture);
+ }
+ if (setTaskLogStep)
+ {
+ props["log.step.MyPerf"] = taskLogStepVal.ToString(CultureInfo.InvariantCulture);
+ }
+ props["directory"] = "RAMDirectory"; // no accidental FS dir.
+ Config config = new Config(props);
+ return new PerfRunData(config);
+ }
+
+ private void doLogStepTest(bool setLogStep, int logStepVal,
+ bool setTaskLogStep, int taskLogStepVal, int expLogStepValue)
+ {
+ PerfRunData runData = createPerfRunData(setLogStep, logStepVal, setTaskLogStep, taskLogStepVal);
+ MyPerfTask mpt = new MyPerfTask(runData);
+ assertEquals(expLogStepValue, mpt.getLogStep());
+ }
+
+ [Test]
+ public void TestLogStep()
+ {
+ doLogStepTest(false, -1, false, -1, PerfTask.DEFAULT_LOG_STEP);
+ doLogStepTest(true, -1, false, -1, int.MaxValue);
+ doLogStepTest(true, 100, false, -1, 100);
+ doLogStepTest(false, -1, true, -1, int.MaxValue);
+ doLogStepTest(false, -1, true, 100, 100);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/SearchWithSortTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/SearchWithSortTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/SearchWithSortTaskTest.cs
new file mode 100644
index 0000000..959681a
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/SearchWithSortTaskTest.cs
@@ -0,0 +1,35 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Search;
+using NUnit.Framework;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class SearchWithSortTaskTest : BenchmarkTestCase
+ {
+ [Test]
+ public void TestSetParams_docField()
+ {
+ SearchWithSortTask task = new SearchWithSortTask(new PerfRunData(new Config(new Dictionary<string, string>())));
+ task.SetParams("doc");
+ assertEquals(SortFieldType.DOC, task.Sort.GetSort()[0].Type);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTaskTest.cs
new file mode 100644
index 0000000..8f710bc
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTaskTest.cs
@@ -0,0 +1,121 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Support;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link WriteEnwikiLineDocTask}.
+ /// </summary>
+ public class WriteEnwikiLineDocTaskTest : BenchmarkTestCase
+ {
+ // class has to be public so that Class.forName.newInstance() will work
+ /** Interleaves category docs with regular docs */
+ public sealed class WriteLineCategoryDocMaker : DocMaker
+ {
+
+ AtomicInt32 flip = new AtomicInt32(0);
+
+ public override Document MakeDocument()
+ {
+ bool isCategory = (flip.IncrementAndGet() % 2 == 0);
+ Document doc = new Document();
+ doc.Add(new StringField(BODY_FIELD, "body text", Field.Store.NO));
+ doc.Add(new StringField(TITLE_FIELD, isCategory ? "Category:title text" : "title text", Field.Store.NO));
+ doc.Add(new StringField(DATE_FIELD, "date text", Field.Store.NO));
+ return doc;
+ }
+
+ }
+
+ private PerfRunData createPerfRunData(FileInfo file, String docMakerName)
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["doc.maker"] = docMakerName;
+ props["line.file.out"] = file.FullName;
+ props["directory"] = "RAMDirectory"; // no accidental FS dir.
+ Config config = new Config(props);
+ return new PerfRunData(config);
+ }
+
+ private void doReadTest(FileInfo file, String expTitle,
+ String expDate, String expBody)
+ {
+ doReadTest(2, file, expTitle, expDate, expBody);
+ FileInfo categoriesFile = WriteEnwikiLineDocTask.CategoriesLineFile(file);
+ doReadTest(2, categoriesFile, "Category:" + expTitle, expDate, expBody);
+ }
+
+ private void doReadTest(int n, FileInfo file, String expTitle, String expDate, String expBody)
+ {
+ Stream @in = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
+ TextReader br = new StreamReader(@in, Encoding.UTF8);
+ try
+ {
+ String line = br.ReadLine();
+ WriteLineDocTaskTest.assertHeaderLine(line);
+ for (int i = 0; i < n; i++)
+ {
+ line = br.ReadLine();
+ assertNotNull(line);
+ String[] parts = line.Split(WriteLineDocTask.SEP).TrimEnd();
+ int numExpParts = expBody == null ? 2 : 3;
+ assertEquals(numExpParts, parts.Length);
+ assertEquals(expTitle, parts[0]);
+ assertEquals(expDate, parts[1]);
+ if (expBody != null)
+ {
+ assertEquals(expBody, parts[2]);
+ }
+ }
+ assertNull(br.ReadLine());
+ }
+ finally
+ {
+ br.Dispose();
+ }
+ }
+
+ [Test]
+ public void TestCategoryLines()
+ {
+ // WriteLineDocTask replaced only \t characters w/ a space, since that's its
+ // separator char. However, it didn't replace newline characters, which
+ // resulted in errors in LineDocSource.
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "two-lines-each.txt"));
+ PerfRunData runData = createPerfRunData(file, typeof(WriteLineCategoryDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteEnwikiLineDocTask(runData);
+ for (int i = 0; i < 4; i++)
+ { // four times so that each file should have 2 lines.
+ wldt.DoLogic();
+ }
+ wldt.Dispose();
+
+
+ doReadTest(file, "title text", "date text", "body text");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteLineDocTaskTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteLineDocTaskTest.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteLineDocTaskTest.cs
new file mode 100644
index 0000000..8edad56
--- /dev/null
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/Tasks/WriteLineDocTaskTest.cs
@@ -0,0 +1,436 @@
+using ICSharpCode.SharpZipLib.BZip2;
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Support;
+using Lucene.Net.Support.Threading;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Text;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the functionality of {@link WriteLineDocTask}.
+ /// </summary>
+ public class WriteLineDocTaskTest : BenchmarkTestCase
+ {
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class WriteLineDocMaker : DocMaker
+ {
+
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ doc.Add(new StringField(BODY_FIELD, "body", Field.Store.NO));
+ doc.Add(new StringField(TITLE_FIELD, "title", Field.Store.NO));
+ doc.Add(new StringField(DATE_FIELD, "date", Field.Store.NO));
+ return doc;
+ }
+
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class NewLinesDocMaker : DocMaker
+ {
+
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ doc.Add(new StringField(BODY_FIELD, "body\r\ntext\ttwo", Field.Store.NO));
+ doc.Add(new StringField(TITLE_FIELD, "title\r\ntext", Field.Store.NO));
+ doc.Add(new StringField(DATE_FIELD, "date\r\ntext", Field.Store.NO));
+ return doc;
+ }
+
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class NoBodyDocMaker : DocMaker
+ {
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ doc.Add(new StringField(TITLE_FIELD, "title", Field.Store.NO));
+ doc.Add(new StringField(DATE_FIELD, "date", Field.Store.NO));
+ return doc;
+ }
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class NoTitleDocMaker : DocMaker
+ {
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ doc.Add(new StringField(BODY_FIELD, "body", Field.Store.NO));
+ doc.Add(new StringField(DATE_FIELD, "date", Field.Store.NO));
+ return doc;
+ }
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class JustDateDocMaker : DocMaker
+ {
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ doc.Add(new StringField(DATE_FIELD, "date", Field.Store.NO));
+ return doc;
+ }
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ // same as JustDate just that this one is treated as legal
+ public sealed class LegalJustDateDocMaker : DocMaker
+ {
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ doc.Add(new StringField(DATE_FIELD, "date", Field.Store.NO));
+ return doc;
+ }
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class EmptyDocMaker : DocMaker
+ {
+ public override Document MakeDocument()
+ {
+ return new Document();
+ }
+ }
+
+ // class has to be public so that Class.forName.newInstance() will work
+ public sealed class ThreadingDocMaker : DocMaker
+ {
+
+ public override Document MakeDocument()
+ {
+ Document doc = new Document();
+ String name = Thread.CurrentThread.Name;
+ doc.Add(new StringField(BODY_FIELD, "body_" + name, Field.Store.NO));
+ doc.Add(new StringField(TITLE_FIELD, "title_" + name, Field.Store.NO));
+ doc.Add(new StringField(DATE_FIELD, "date_" + name, Field.Store.NO));
+ return doc;
+ }
+
+ }
+
+ private PerfRunData createPerfRunData(FileInfo file,
+ bool allowEmptyDocs,
+ String docMakerName)
+ {
+ Dictionary<string, string> props = new Dictionary<string, string>();
+ props["doc.maker"] = docMakerName;
+ props["line.file.out"] = file.FullName;
+ props["directory"] = "RAMDirectory"; // no accidental FS dir.
+ if (allowEmptyDocs)
+ {
+ props["sufficient.fields"] = ",";
+ }
+ if (typeof(LegalJustDateDocMaker).Equals(Type.GetType(docMakerName)))
+ {
+ props["line.fields"] = DocMaker.DATE_FIELD;
+ props["sufficient.fields"] = DocMaker.DATE_FIELD;
+ }
+ Config config = new Config(props);
+ return new PerfRunData(config);
+ }
+
+ private void doReadTest(FileInfo file, FileType fileType, String expTitle,
+ String expDate, String expBody)
+ {
+ Stream input = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
+ switch (fileType)
+ {
+ case FileType.BZIP2:
+ input = new BZip2InputStream(input);
+ break;
+ case FileType.GZIP:
+ input = new GZipStream(input, CompressionMode.Decompress);
+ break;
+ case FileType.PLAIN:
+ break; // nothing to do
+ default:
+ assertFalse("Unknown file type!", true); //fail, should not happen
+ break;
+ }
+ TextReader br = new StreamReader(input, Encoding.UTF8);
+ try
+ {
+ String line = br.ReadLine();
+ assertHeaderLine(line);
+ line = br.ReadLine();
+ assertNotNull(line);
+ String[] parts = line.Split(WriteLineDocTask.SEP).TrimEnd();
+ int numExpParts = expBody == null ? 2 : 3;
+ assertEquals(numExpParts, parts.Length);
+ assertEquals(expTitle, parts[0]);
+ assertEquals(expDate, parts[1]);
+ if (expBody != null)
+ {
+ assertEquals(expBody, parts[2]);
+ }
+ assertNull(br.ReadLine());
+ }
+ finally
+ {
+ br.Dispose();
+ }
+ }
+
+ internal static void assertHeaderLine(String line)
+ {
+ assertTrue("First line should be a header line", line.StartsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR, StringComparison.Ordinal));
+ }
+
+ /* Tests WriteLineDocTask with a bzip2 format. */
+ [Test]
+ public void TestBZip2()
+ {
+
+ // Create a document in bz2 format.
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line.bz2"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(WriteLineDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+
+ doReadTest(file, FileType.BZIP2, "title", "date", "body");
+ }
+
+ /* Tests WriteLineDocTask with a gzip format. */
+ [Test]
+ public void TestGZip()
+ {
+
+ // Create a document in gz format.
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line.gz"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(WriteLineDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+
+ doReadTest(file, FileType.GZIP, "title", "date", "body");
+ }
+
+ [Test]
+ public void TestRegularFile()
+ {
+
+ // Create a document in regular format.
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(WriteLineDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+
+ doReadTest(file, FileType.PLAIN, "title", "date", "body");
+ }
+
+ [Test]
+ public void TestCharsReplace()
+ {
+ // WriteLineDocTask replaced only \t characters w/ a space, since that's its
+ // separator char. However, it didn't replace newline characters, which
+ // resulted in errors in LineDocSource.
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(NewLinesDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+
+ doReadTest(file, FileType.PLAIN, "title text", "date text", "body text two");
+ }
+
+ [Test]
+ public void TestEmptyBody()
+ {
+ // WriteLineDocTask threw away documents w/ no BODY element, even if they
+ // had a TITLE element (LUCENE-1755). It should throw away documents if they
+ // don't have BODY nor TITLE
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(NoBodyDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+
+ doReadTest(file, FileType.PLAIN, "title", "date", null);
+ }
+
+ [Test]
+ public void TestEmptyTitle()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(NoTitleDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+
+ doReadTest(file, FileType.PLAIN, "", "date", "body");
+ }
+
+ /** Fail by default when there's only date */
+ [Test]
+ public void TestJustDate()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(JustDateDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+ TextReader br = new StreamReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8);
+ try
+ {
+ String line = br.ReadLine();
+ assertHeaderLine(line);
+ line = br.ReadLine();
+ assertNull(line);
+ }
+ finally
+ {
+ br.Dispose();
+ }
+ }
+
+ [Test]
+ public void TestLegalJustDate()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(LegalJustDateDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+ TextReader br = new StreamReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8);
+ try
+ {
+ String line = br.ReadLine();
+ assertHeaderLine(line);
+ line = br.ReadLine();
+ assertNotNull(line);
+ }
+ finally
+ {
+ br.Dispose();
+ }
+ }
+
+ [Test]
+ public void TestEmptyDoc()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, true, typeof(EmptyDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ wldt.DoLogic();
+ wldt.Dispose();
+
+ TextReader br = new StreamReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8);
+ try
+ {
+ String line = br.ReadLine();
+ assertHeaderLine(line);
+ line = br.ReadLine();
+ assertNotNull(line);
+ }
+ finally
+ {
+ br.Dispose();
+ }
+ }
+ private class ThreadAnonymousHelper : ThreadClass
+ {
+ private readonly WriteLineDocTask wldt;
+ public ThreadAnonymousHelper(string name, WriteLineDocTask wldt)
+ : base(name)
+ {
+ this.wldt = wldt;
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ wldt.DoLogic();
+ }
+ catch (Exception e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+ }
+
+ [Test]
+ public void TestMultiThreaded()
+ {
+ FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
+ PerfRunData runData = createPerfRunData(file, false, typeof(ThreadingDocMaker).AssemblyQualifiedName);
+ WriteLineDocTask wldt = new WriteLineDocTask(runData);
+ ThreadClass[] threads = new ThreadClass[10];
+ for (int i = 0; i < threads.Length; i++)
+ {
+ threads[i] = new ThreadAnonymousHelper("t" + i, wldt);
+ }
+
+ foreach (ThreadClass t in threads) t.Start();
+ foreach (ThreadClass t in threads) t.Join();
+
+ wldt.Dispose();
+
+ ISet<String> ids = new HashSet<string>();
+ TextReader br = new StreamReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read), Encoding.UTF8);
+ try
+ {
+ String line = br.ReadLine();
+ assertHeaderLine(line); // header line is written once, no matter how many threads there are
+ for (int i = 0; i < threads.Length; i++)
+ {
+ line = br.ReadLine();
+ String[] parts = line.Split(WriteLineDocTask.SEP).TrimEnd();
+ assertEquals(3, parts.Length);
+ // check that all thread names written are the same in the same line
+ String tname = parts[0].Substring(parts[0].IndexOf('_'));
+ ids.add(tname);
+ assertEquals(tname, parts[1].Substring(parts[1].IndexOf('_')));
+ assertEquals(tname, parts[2].Substring(parts[2].IndexOf('_')));
+ }
+ // only threads.length lines should exist
+ assertNull(br.ReadLine());
+ assertEquals(threads.Length, ids.size());
+ }
+ finally
+ {
+ br.Dispose();
+ }
+ }
+ }
+}
[09/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs b/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs
new file mode 100644
index 0000000..ed54d92
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs
@@ -0,0 +1,108 @@
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Stats
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test run data points collected as the test proceeds.
+ /// </summary>
+ public class Points
+ {
+ // stat points ordered by their start time.
+ // for now we collect points as TaskStats objects.
+ // later might optimize to collect only native data.
+ private List<TaskStats> points = new List<TaskStats>();
+
+ private int nextTaskRunNum = 0;
+
+ private TaskStats currentStats;
+
+ /// <summary>
+ /// Create a Points statistics object.
+ /// </summary>
+ public Points(Config config)
+ {
+ }
+
+ /// <summary>
+ /// Gets the current task stats.
+ /// The actual task stats are returned, so caller should not modify this task stats.
+ /// </summary>
+ public virtual IList<TaskStats> TaskStats
+ {
+ get { return points; }
+ }
+
+ /// <summary>
+ /// Mark that a task is starting.
+ /// Create a task stats for it and store it as a point.
+ /// </summary>
+ /// <param name="task">The starting task.</param>
+ /// <param name="round">The new task stats created for the starting task.</param>
+ /// <returns></returns>
+ public virtual TaskStats MarkTaskStart(PerfTask task, int round)
+ {
+ lock (this)
+ {
+ TaskStats stats = new TaskStats(task, NextTaskRunNum(), round);
+ this.currentStats = stats;
+ points.Add(stats);
+ return stats;
+ }
+ }
+
+ public virtual TaskStats CurrentStats
+ {
+ get { return currentStats; }
+ }
+
+ // return next task num
+ private int NextTaskRunNum()
+ {
+ lock (this)
+ {
+ return nextTaskRunNum++;
+ }
+ }
+
+ /// <summary>
+ /// mark the end of a task
+ /// </summary>
+ public virtual void MarkTaskEnd(TaskStats stats, int count)
+ {
+ lock (this)
+ {
+ int numParallelTasks = nextTaskRunNum - 1 - stats.TaskRunNum;
+ // note: if the stats were cleared, might be that this stats object is
+ // no longer in points, but this is just ok.
+ stats.MarkEnd(numParallelTasks, count);
+ }
+ }
+
+ /// <summary>
+ /// Clear all data, prepare for more tests.
+ /// </summary>
+ public virtual void ClearData()
+ {
+ points.Clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs b/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs
new file mode 100644
index 0000000..59fd725
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs
@@ -0,0 +1,70 @@
+namespace Lucene.Net.Benchmarks.ByTask.Stats
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Textual report of current statistics.
+ /// </summary>
+ public class Report
+ {
+ private string text;
+ private int size;
+ private int outOf;
+ private int reported;
+
+ public Report(string text, int size, int reported, int outOf)
+ {
+ this.text = text;
+ this.size = size;
+ this.reported = reported;
+ this.outOf = outOf;
+ }
+
+ /// <summary>
+ /// Gets total number of stats points when this report was created.
+ /// </summary>
+ public virtual int OutOf
+ {
+ get { return outOf; }
+ }
+
+ /// <summary>
+ /// Gets number of lines in the report.
+ /// </summary>
+ public virtual int Count
+ {
+ get { return size; }
+ }
+
+ /// <summary>
+ /// Gets the report text.
+ /// </summary>
+ public virtual string Text
+ {
+ get { return text; }
+ }
+
+ /// <summary>
+ /// Gets number of stats points represented in this report.
+ /// </summary>
+ public virtual int Reported
+ {
+ get { return reported; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs b/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs
new file mode 100644
index 0000000..4d32c7b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs
@@ -0,0 +1,237 @@
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Support;
+using System;
+using System.Diagnostics;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Stats
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Statistics for a task run.
+ /// <para/>
+ /// The same task can run more than once, but, if that task records statistics,
+ /// each run would create its own TaskStats.
+ /// </summary>
+ public class TaskStats
+ {
+ /// <summary>Task for which data was collected.</summary>
+ private PerfTask task;
+
+ /// <summary>Round in which task run started.</summary>
+ private int round;
+
+ /// <summary>Task start time.</summary>
+ private long start;
+
+ /// <summary>Task elapsed time. elapsed >= 0 indicates run completion!</summary>
+ private long elapsed = -1;
+
+ /// <summary>Max tot mem during task.</summary>
+ private long maxTotMem;
+
+ /// <summary>Max used mem during task.</summary>
+ private long maxUsedMem;
+
+ /// <summary>Serial run number of this task run in the perf run.</summary>
+ private int taskRunNum;
+
+ /// <summary>Number of other tasks that started to run while this task was still running.</summary>
+ private int numParallelTasks;
+
+ /// <summary>
+ /// Number of work items done by this task.
+ /// For indexing that can be number of docs added.
+ /// For warming that can be number of scanned items, etc.
+ /// For repeating tasks, this is a sum over repetitions.
+ /// </summary>
+ private int count;
+
+ /// <summary>
+ /// Number of similar tasks aggregated into this record.
+ /// Used when summing up on few runs/instances of similar tasks.
+ /// </summary>
+ private int numRuns = 1;
+
+ /// <summary>
+ /// Create a run data for a task that is starting now.
+ /// To be called from Points.
+ /// </summary>
+ internal TaskStats(PerfTask task, int taskRunNum, int round)
+ {
+ this.task = task;
+ this.taskRunNum = taskRunNum;
+ this.round = round;
+ maxTotMem = GC.GetTotalMemory(false); //Runtime.getRuntime().totalMemory();
+ maxUsedMem = maxTotMem; // - Runtime.getRuntime().freeMemory(); // LUCENENET TODO: available RAM
+ start = Stopwatch.GetTimestamp();
+ }
+
+ /// <summary>
+ /// Mark the end of a task.
+ /// </summary>
+ internal void MarkEnd(int numParallelTasks, int count)
+ {
+ elapsed = Support.Time.CurrentTimeMilliseconds();
+ long totMem = GC.GetTotalMemory(false); //Runtime.getRuntime().totalMemory();
+ if (totMem > maxTotMem)
+ {
+ maxTotMem = totMem;
+ }
+ long usedMem = totMem; //- Runtime.getRuntime().freeMemory(); // LUCENENET TODO: available RAM
+ if (usedMem > maxUsedMem)
+ {
+ maxUsedMem = usedMem;
+ }
+ this.numParallelTasks = numParallelTasks;
+ this.count = count;
+ }
+
+ private int[] countsByTime;
+ private long countsByTimeStepMSec;
+
+ public virtual void SetCountsByTime(int[] counts, long msecStep)
+ {
+ countsByTime = counts;
+ countsByTimeStepMSec = msecStep;
+ }
+
+ [WritableArray]
+ public virtual int[] GetCountsByTime()
+ {
+ return countsByTime;
+ }
+
+ public virtual long CountsByTimeStepMSec
+ {
+ get { return countsByTimeStepMSec; }
+ }
+
+ /// <summary>Gets the taskRunNum.</summary>
+ public virtual int TaskRunNum
+ {
+ get { return taskRunNum; }
+ }
+
+ /// <seealso cref="object.ToString()"/>
+ public override string ToString()
+ {
+ StringBuilder res = new StringBuilder(task.GetName());
+ res.Append(" ");
+ res.Append(count);
+ res.Append(" ");
+ res.Append(elapsed);
+ return res.ToString();
+ }
+
+ /// <summary>Gets the count.</summary>
+ public virtual int Count
+ {
+ get { return count; }
+ }
+
+ /// <summary>Gets elapsed time.</summary>
+ public virtual long Elapsed
+ {
+ get { return elapsed; }
+ }
+
+ /// <summary>Gets the maxTotMem.</summary>
+ public virtual long MaxTotMem
+ {
+ get { return maxTotMem; }
+ }
+
+ /// <summary>Gets the maxUsedMem.</summary>
+ public virtual long MaxUsedMem
+ {
+ get { return maxUsedMem; }
+ }
+
+ /// <summary>Gets the numParallelTasks.</summary>
+ public virtual int NumParallelTasks
+ {
+ get { return numParallelTasks; }
+ }
+
+ /// <summary>Gets the task.</summary>
+ public virtual PerfTask Task
+ {
+ get { return task; }
+ }
+
+ /// <summary>Gets the numRuns.</summary>
+ public virtual int NumRuns
+ {
+ get { return numRuns; }
+ }
+
+ /// <summary>
+ /// Add data from another stat, for aggregation.
+ /// </summary>
+ /// <param name="stat2">The added stat data.</param>
+ public virtual void Add(TaskStats stat2)
+ {
+ numRuns += stat2.NumRuns;
+ elapsed += stat2.Elapsed;
+ maxTotMem += stat2.MaxTotMem;
+ maxUsedMem += stat2.MaxUsedMem;
+ count += stat2.Count;
+ if (round != stat2.round)
+ {
+ round = -1; // no meaning if aggregating tasks of different round.
+ }
+
+ if (countsByTime != null && stat2.countsByTime != null)
+ {
+ if (countsByTimeStepMSec != stat2.countsByTimeStepMSec)
+ {
+ throw new InvalidOperationException("different by-time msec step");
+ }
+ if (countsByTime.Length != stat2.countsByTime.Length)
+ {
+ throw new InvalidOperationException("different by-time msec count");
+ }
+ for (int i = 0; i < stat2.countsByTime.Length; i++)
+ {
+ countsByTime[i] += stat2.countsByTime[i];
+ }
+ }
+ }
+
+#if FEATURE_CLONEABLE
+ /// <seealso cref="ICloneable.Clone()"/>
+#endif
+ public virtual object Clone()
+ {
+ TaskStats c = (TaskStats)base.MemberwiseClone();
+ if (c.countsByTime != null)
+ {
+ c.countsByTime = (int[])c.countsByTime.Clone();
+ }
+ return c;
+ }
+
+ /// <summary>Gets the round number.</summary>
+ public virtual int Round
+ {
+ get { return round; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs
new file mode 100644
index 0000000..8e92f48
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs
@@ -0,0 +1,93 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Documents;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Add a document, optionally of a certain size.
+ /// <para/>
+ /// Other side effects: none.
+ /// <para/>
+ /// Takes optional param: document size.
+ /// </summary>
+ public class AddDocTask : PerfTask
+ {
+ public AddDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ private int docSize = 0;
+
+ /// <summary>
+ /// Volatile data passed between <see cref="Setup()"/>, <see cref="DoLogic()"/>, <see cref="TearDown()"/>.
+ /// The doc is created at <see cref="Setup()"/> and added at <see cref="DoLogic()"/>.
+ /// </summary>
+ protected Document m_doc = null;
+
+ public override void Setup()
+ {
+ base.Setup();
+ DocMaker docMaker = RunData.DocMaker;
+ if (docSize > 0)
+ {
+ m_doc = docMaker.MakeDocument(docSize);
+ }
+ else
+ {
+ m_doc = docMaker.MakeDocument();
+ }
+ }
+
+ public override void TearDown()
+ {
+ m_doc = null;
+ base.TearDown();
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return string.Format(CultureInfo.InvariantCulture, "added {0:N9} docs", recsCount);
+ }
+
+ public override int DoLogic()
+ {
+ RunData.IndexWriter.AddDocument(m_doc);
+ return 1;
+ }
+
+ /// <summary>
+ /// Set the params (docSize only).
+ /// </summary>
+ /// <param name="params">docSize, or 0 for no limit.</param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ docSize = (int)float.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs
new file mode 100644
index 0000000..6ae761c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs
@@ -0,0 +1,95 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Facet;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Add a faceted document.
+ /// </summary>
+ /// <remarks>
+ /// Config properties:
+ /// <list type="bullet">
+ /// <item>
+ /// <term>with.facets</term>
+ /// <description>
+ /// <tells whether to actually add any facets to the document| Default: true>
+ /// <para/>
+ /// This config property allows to easily compare the performance of adding docs
+ /// with and without facets. Note that facets are created even when this is
+ /// <c>false</c>, just that they are not added to the document (nor to the taxonomy).
+ /// </description>
+ /// </item>
+ /// </list>
+ /// <para/>
+ /// See <see cref="AddDocTask"/> for general document parameters and configuration.
+ /// <para/>
+ /// Makes use of the <see cref="FacetSource"/> in effect - see <see cref="PerfRunData"/> for
+ /// facet source settings.
+ /// </remarks>
+ public class AddFacetedDocTask : AddDocTask
+ {
+ private FacetsConfig config;
+
+ public AddFacetedDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void Setup()
+ {
+ base.Setup();
+ if (config == null)
+ {
+ bool withFacets = RunData.Config.Get("with.facets", true);
+ if (withFacets)
+ {
+ FacetSource facetsSource = RunData.FacetSource;
+ config = new FacetsConfig();
+ facetsSource.Configure(config);
+ }
+ }
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ if (config == null)
+ {
+ return base.GetLogMessage(recsCount);
+ }
+ return base.GetLogMessage(recsCount) + " with facets";
+ }
+
+ public override int DoLogic()
+ {
+ if (config != null)
+ {
+ List<FacetField> facets = new List<FacetField>();
+ RunData.FacetSource.GetNextFacets(facets);
+ foreach (FacetField ff in facets)
+ {
+ m_doc.Add(ff);
+ }
+ m_doc = config.Build(RunData.TaxonomyWriter, m_doc);
+ }
+ return base.DoLogic();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs
new file mode 100644
index 0000000..f05ca32
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs
@@ -0,0 +1,104 @@
+using Lucene.Net.Index;
+using Lucene.Net.Store;
+using System;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Adds an input index to an existing index, using
+ /// <see cref="IndexWriter.AddIndexes(Store.Directory[])"/> or
+ /// <see cref="IndexWriter.AddIndexes(IndexReader[])"/>. The location of the input
+ /// index is specified by the parameter <see cref="ADDINDEXES_INPUT_DIR"/> and is
+ /// assumed to be a directory on the file system.
+ /// <para/>
+ /// Takes optional parameter <see cref="useAddIndexesDir"/> which specifies which
+ /// AddIndexes variant to use (defaults to <c>true</c>, to use <c>AddIndexes(Directory)</c>).
+ /// </summary>
+ public class AddIndexesTask : PerfTask
+ {
+ public static readonly string ADDINDEXES_INPUT_DIR = "addindexes.input.dir";
+
+ public AddIndexesTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ private bool useAddIndexesDir = true;
+ private FSDirectory inputDir;
+
+ public override void Setup()
+ {
+ base.Setup();
+ string inputDirProp = RunData.Config.Get(ADDINDEXES_INPUT_DIR, null);
+ if (inputDirProp == null)
+ {
+ throw new ArgumentException("config parameter " + ADDINDEXES_INPUT_DIR + " not specified in configuration");
+ }
+ inputDir = FSDirectory.Open(new DirectoryInfo(inputDirProp));
+ }
+
+ public override int DoLogic()
+ {
+ IndexWriter writer = RunData.IndexWriter;
+ if (useAddIndexesDir)
+ {
+ writer.AddIndexes(inputDir);
+ }
+ else
+ {
+ IndexReader r = DirectoryReader.Open(inputDir);
+ try
+ {
+ writer.AddIndexes(r);
+ }
+ finally
+ {
+ r.Dispose();
+ }
+ }
+ return 1;
+ }
+
+ /// <summary>
+ /// Set the params (useAddIndexesDir only)
+ /// </summary>
+ /// <param name="params">
+ /// <c>useAddIndexesDir=true</c> for using <see cref="IndexWriter.AddIndexes(Store.Directory[])"/> or <c>false</c> for
+ /// using <see cref="IndexWriter.AddIndexes(IndexReader[])"/>. Defaults to <c>true</c>.
+ /// </param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ useAddIndexesDir = bool.Parse(@params);
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+
+ public override void TearDown()
+ {
+ inputDir.Dispose();
+ base.TearDown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
new file mode 100644
index 0000000..56b0114
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs
@@ -0,0 +1,580 @@
+using Lucene.Net.Analysis.Util;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support.IO;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text.RegularExpressions;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Analyzer factory construction task. The name given to the constructed factory may
+ /// be given to <see cref="NewAnalyzerTask"/>, which will call <see cref="AnalyzerFactory.Create()"/>.
+ /// </summary>
+ /// <remarks>
+ /// Params are in the form argname:argvalue or argname:"argvalue" or argname:'argvalue';
+ /// use backslashes to escape '"' or "'" inside a quoted value when it's used as the enclosing
+ /// quotation mark,
+ /// <para/>
+ /// Specify params in a comma separated list of the following, in order:
+ /// <list type="number">
+ /// <item><description>
+ /// <list type="bullet">
+ /// <item><description><b>Required</b>: <c>name:<i>analyzer-factory-name</i></c></description></item>
+ /// <item><description>Optional: <c>positionIncrementGap:<i>int value</i></c> (default: 0)</description></item>
+ /// <item><description>Optional: <c>offsetGap:<i>int value</i></c> (default: 1)</description></item>
+ /// </list>
+ /// </description></item>
+ /// <item><description>zero or more CharFilterFactory's, followed by</description></item>
+ /// <item><description>exactly one TokenizerFactory, followed by</description></item>
+ /// <item><description>zero or more TokenFilterFactory's</description></item>
+ /// </list>
+ /// <para/>
+ /// Each component analysis factory map specify <tt>luceneMatchVersion</tt> (defaults to
+ /// <see cref="LuceneVersion.LUCENE_CURRENT"/>) and any of the args understood by the specified
+ /// *Factory class, in the above-describe param format.
+ /// <para/>
+ /// Example:
+ /// <code>
+ /// -AnalyzerFactory(name:'strip html, fold to ascii, whitespace tokenize, max 10k tokens',
+ /// positionIncrementGap:100,
+ /// HTMLStripCharFilter,
+ /// MappingCharFilter(mapping:'mapping-FoldToASCII.txt'),
+ /// WhitespaceTokenizer(luceneMatchVersion:LUCENE_43),
+ /// TokenLimitFilter(maxTokenCount:10000, consumeAllTokens:false))
+ /// [...]
+ /// -NewAnalyzer('strip html, fold to ascii, whitespace tokenize, max 10k tokens')
+ /// </code>
+ /// <para/>
+ /// <see cref="AnalyzerFactory"/> will direct analysis component factories to look for resources
+ /// under the directory specified in the "work.dir" property.
+ /// </remarks>
+ public class AnalyzerFactoryTask : PerfTask
+ {
+ private static readonly string LUCENE_ANALYSIS_PACKAGE_PREFIX = "Lucene.Net.Analysis.";
+ private static readonly Regex ANALYSIS_COMPONENT_SUFFIX_PATTERN
+ = new Regex("(?s:(?:(?:Token|Char)?Filter|Tokenizer)(?:Factory)?)$", RegexOptions.Compiled);
+ private static readonly Regex TRAILING_DOT_ZERO_PATTERN = new Regex(@"\.0$", RegexOptions.Compiled);
+
+ private enum ArgType { ANALYZER_ARG, ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER, TOKENFILTER }
+
+ string factoryName = null;
+ int? positionIncrementGap = null;
+ int? offsetGap = null;
+ private IList<CharFilterFactory> charFilterFactories = new List<CharFilterFactory>();
+ private TokenizerFactory tokenizerFactory = null;
+ private IList<TokenFilterFactory> tokenFilterFactories = new List<TokenFilterFactory>();
+
+ public AnalyzerFactoryTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ return 1;
+ }
+
+ /// <summary>
+ /// Sets the params.
+ /// Analysis component factory names may optionally include the "Factory" suffix.
+ /// </summary>
+ /// <param name="params">
+ /// analysis pipeline specification: name, (optional) positionIncrementGap,
+ /// (optional) offsetGap, 0+ CharFilterFactory's, 1 TokenizerFactory,
+ /// and 0+ TokenFilterFactory's
+ /// </param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ ArgType expectedArgType = ArgType.ANALYZER_ARG;
+
+ StreamTokenizer stok = new StreamTokenizer(new StringReader(@params));
+ stok.CommentChar('#');
+ stok.QuoteChar('"');
+ stok.QuoteChar('\'');
+ stok.IsEOLSignificant = false;
+ stok.OrdinaryChar('(');
+ stok.OrdinaryChar(')');
+ stok.OrdinaryChar(':');
+ stok.OrdinaryChar(',');
+ try
+ {
+ while (stok.NextToken() != StreamTokenizer.TT_EOF)
+ {
+ switch (stok.TokenType)
+ {
+ case ',':
+ {
+ // Do nothing
+ break;
+ }
+ case StreamTokenizer.TT_WORD:
+ {
+ if (expectedArgType.Equals(ArgType.ANALYZER_ARG))
+ {
+ string argName = stok.StringValue;
+ if (!argName.Equals("name", StringComparison.OrdinalIgnoreCase)
+ && !argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase)
+ && !argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase))
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Missing 'name' param to AnalyzerFactory: '" + @params + "'");
+ }
+ stok.NextToken();
+ if (stok.TokenType != ':')
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to AnalyzerFactory");
+ }
+
+ stok.NextToken();
+ string argValue = stok.StringValue;
+ switch (stok.TokenType)
+ {
+ case StreamTokenizer.TT_NUMBER:
+ {
+ argValue = stok.NumberValue.ToString(CultureInfo.InvariantCulture);
+ // Drop the ".0" from numbers, for integer arguments
+ argValue = TRAILING_DOT_ZERO_PATTERN.Replace(argValue, "", 1);
+ // Intentional fallthrough
+
+ if (argName.Equals("name", StringComparison.OrdinalIgnoreCase))
+ {
+ factoryName = argValue;
+ expectedArgType = ArgType.ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER;
+ }
+ else
+ {
+ int intArgValue = 0;
+ try
+ {
+ intArgValue = int.Parse(argValue, CultureInfo.InvariantCulture);
+ }
+ catch (FormatException e)
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Exception parsing " + argName + " value '" + argValue + "'", e);
+ }
+ if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase))
+ {
+ positionIncrementGap = intArgValue;
+ }
+ else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase))
+ {
+ offsetGap = intArgValue;
+ }
+ }
+ break;
+ }
+ case '"':
+ case '\'':
+ case StreamTokenizer.TT_WORD:
+ {
+ if (argName.Equals("name", StringComparison.OrdinalIgnoreCase))
+ {
+ factoryName = argValue;
+ expectedArgType = ArgType.ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER;
+ }
+ else
+ {
+ int intArgValue = 0;
+ try
+ {
+ intArgValue = int.Parse(argValue, CultureInfo.InvariantCulture);
+ }
+ catch (FormatException e)
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Exception parsing " + argName + " value '" + argValue + "'", e);
+ }
+ if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase))
+ {
+ positionIncrementGap = intArgValue;
+ }
+ else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase))
+ {
+ offsetGap = intArgValue;
+ }
+ }
+ break;
+ }
+ case StreamTokenizer.TT_EOF:
+ {
+ throw new Exception("Unexpected EOF: " + stok.ToString());
+ }
+ default:
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString());
+ }
+ }
+ }
+ else if (expectedArgType.Equals(ArgType.ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER))
+ {
+ string argName = stok.StringValue;
+
+ if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase)
+ || argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase))
+ {
+ stok.NextToken();
+ if (stok.TokenType != ':')
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to AnalyzerFactory");
+ }
+ stok.NextToken();
+ int intArgValue = (int)stok.NumberValue;
+ switch (stok.TokenType)
+ {
+ case '"':
+ case '\'':
+ case StreamTokenizer.TT_WORD:
+ {
+ intArgValue = 0;
+ try
+ {
+ intArgValue = int.Parse(stok.StringValue.Trim(), CultureInfo.InvariantCulture);
+ }
+ catch (FormatException e)
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Exception parsing " + argName + " value '" + stok.StringValue + "'", e);
+ }
+ // Intentional fall-through
+
+ if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase))
+ {
+ positionIncrementGap = intArgValue;
+ }
+ else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase))
+ {
+ offsetGap = intArgValue;
+ }
+ break;
+ }
+ case StreamTokenizer.TT_NUMBER:
+ {
+ if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase))
+ {
+ positionIncrementGap = intArgValue;
+ }
+ else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase))
+ {
+ offsetGap = intArgValue;
+ }
+ break;
+ }
+ case StreamTokenizer.TT_EOF:
+ {
+ throw new Exception("Unexpected EOF: " + stok.ToString());
+ }
+ default:
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString());
+ }
+ }
+ break;
+ }
+ try
+ {
+ Type clazz;
+ clazz = LookupAnalysisClass(argName, typeof(CharFilterFactory));
+ CreateAnalysisPipelineComponent(stok, clazz);
+ }
+ catch (ArgumentException /*e*/)
+ {
+ try
+ {
+ Type clazz;
+ clazz = LookupAnalysisClass(argName, typeof(TokenizerFactory));
+ CreateAnalysisPipelineComponent(stok, clazz);
+ expectedArgType = ArgType.TOKENFILTER;
+ }
+ catch (ArgumentException e2)
+ {
+ throw new Exception("Line #" + GetLineNumber(stok) + ": Can't find class '"
+ + argName + "' as CharFilterFactory or TokenizerFactory", e2);
+ }
+ }
+ }
+ else
+ { // expectedArgType = ArgType.TOKENFILTER
+ string className = stok.StringValue;
+ Type clazz;
+ try
+ {
+ clazz = LookupAnalysisClass(className, typeof(TokenFilterFactory));
+ }
+ catch (ArgumentException e)
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Can't find class '" + className + "' as TokenFilterFactory", e);
+ }
+ CreateAnalysisPipelineComponent(stok, clazz);
+ }
+ break;
+ }
+ default:
+ {
+ throw new Exception("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString());
+ }
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ if (e.Message.StartsWith("Line #", StringComparison.Ordinal))
+ {
+ throw e;
+ }
+ else
+ {
+ throw new Exception("Line #" + GetLineNumber(stok) + ": ", e);
+ }
+ }
+
+ AnalyzerFactory analyzerFactory = new AnalyzerFactory
+ (charFilterFactories, tokenizerFactory, tokenFilterFactories);
+ analyzerFactory.SetPositionIncrementGap(positionIncrementGap.GetValueOrDefault());
+ analyzerFactory.SetOffsetGap(offsetGap.GetValueOrDefault());
+ RunData.AnalyzerFactories[factoryName] = analyzerFactory;
+ }
+
+ /// <summary>
+ /// Instantiates the given analysis factory class after pulling params from
+ /// the given stream tokenizer, then stores the result in the appropriate
+ /// pipeline component list.
+ /// </summary>
+ /// <param name="stok">Stream tokenizer from which to draw analysis factory params.</param>
+ /// <param name="clazz">Analysis factory class to instantiate.</param>
+ private void CreateAnalysisPipelineComponent(StreamTokenizer stok, Type clazz)
+ {
+ IDictionary<string, string> argMap = new Dictionary<string, string>();
+ bool parenthetical = false;
+ try
+ {
+ while (stok.NextToken() != StreamTokenizer.TT_EOF)
+ {
+ switch (stok.TokenType)
+ {
+ case ',':
+ {
+ if (parenthetical)
+ {
+ // Do nothing
+ break;
+ }
+ else
+ {
+ // Finished reading this analysis factory configuration
+ goto WHILE_LOOP_BREAK;
+ }
+ }
+ case '(':
+ {
+ if (parenthetical)
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Unexpected opening parenthesis.");
+ }
+ parenthetical = true;
+ break;
+ }
+ case ')':
+ {
+ if (parenthetical)
+ {
+ parenthetical = false;
+ }
+ else
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Unexpected closing parenthesis.");
+ }
+ break;
+ }
+ case StreamTokenizer.TT_WORD:
+ {
+ if (!parenthetical)
+ {
+ throw new Exception("Line #" + GetLineNumber(stok) + ": Unexpected token '" + stok.StringValue + "'");
+ }
+ string argName = stok.StringValue;
+ stok.NextToken();
+ if (stok.TokenType != ':')
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to " + clazz.Name);
+ }
+ stok.NextToken();
+ string argValue = stok.StringValue;
+ switch (stok.TokenType)
+ {
+ case StreamTokenizer.TT_NUMBER:
+ {
+ argValue = stok.NumberValue.ToString(CultureInfo.InvariantCulture);
+ // Drop the ".0" from numbers, for integer arguments
+ argValue = TRAILING_DOT_ZERO_PATTERN.Replace(argValue, "", 1);
+ // Intentional fall-through
+ argMap[argName] = argValue;
+ break;
+ }
+ case '"':
+ case '\'':
+ case StreamTokenizer.TT_WORD:
+ {
+ argMap[argName] = argValue;
+ break;
+ }
+ case StreamTokenizer.TT_EOF:
+ {
+ throw new Exception("Unexpected EOF: " + stok.ToString());
+ }
+ default:
+ {
+ throw new Exception
+ ("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString());
+ }
+ }
+ break;
+ }
+ }
+ }
+ WHILE_LOOP_BREAK: { }
+
+ if (!argMap.ContainsKey("luceneMatchVersion"))
+ {
+#pragma warning disable 612, 618
+ argMap["luceneMatchVersion"] = LuceneVersion.LUCENE_CURRENT.ToString();
+#pragma warning restore 612, 618
+ }
+ AbstractAnalysisFactory instance;
+ try
+ {
+ instance = (AbstractAnalysisFactory)Activator.CreateInstance(clazz, argMap);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("Line #" + GetLineNumber(stok) + ": ", e);
+ }
+ if (instance is IResourceLoaderAware)
+ {
+ DirectoryInfo baseDir = new DirectoryInfo(RunData.Config.Get("work.dir", "work"));
+ ((IResourceLoaderAware)instance).Inform(new FilesystemResourceLoader(baseDir));
+ }
+ if (typeof(CharFilterFactory).IsAssignableFrom(clazz))
+ {
+ charFilterFactories.Add((CharFilterFactory)instance);
+ }
+ else if (typeof(TokenizerFactory).IsAssignableFrom(clazz))
+ {
+ tokenizerFactory = (TokenizerFactory)instance;
+ }
+ else if (typeof(TokenFilterFactory).IsAssignableFrom(clazz))
+ {
+ tokenFilterFactories.Add((TokenFilterFactory)instance);
+ }
+ }
+ catch (Exception e)
+ {
+ if (e.Message.StartsWith("Line #", StringComparison.Ordinal))
+ {
+ throw e;
+ }
+ else
+ {
+ throw new Exception("Line #" + GetLineNumber(stok) + ": ", e);
+ }
+ }
+ }
+
+ /// <summary>
+ /// This method looks up a class with its fully qualified name (FQN), or a short-name
+ /// class-simplename, or with a package suffix, assuming "Lucene.Net.Analysis."
+ /// as the namespace prefix (e.g. "standard.ClassicTokenizerFactory" ->
+ /// "Lucene.Net.Analysis.Standard.ClassicTokenizerFactory").
+ /// </summary>
+ /// <remarks>
+ /// If <paramref name="className"/> contains a period, the class is first looked up as-is, assuming that it
+ /// is an FQN. If this fails, lookup is retried after prepending the Lucene analysis
+ /// package prefix to the class name.
+ /// <para/>
+ /// If <paramref name="className"/> does not contain a period, the analysis SPI *Factory.LookupClass()
+ /// methods are used to find the class.
+ /// </remarks>
+ /// <param name="className">The namespace qualified name or the short name of the class.</param>
+ /// <param name="expectedType">The superclass <paramref name="className"/> is expected to extend. </param>
+ /// <returns>The loaded type.</returns>
+ /// <exception cref="TypeLoadException">If lookup fails.</exception>
+ public virtual Type LookupAnalysisClass(string className, Type expectedType)
+ {
+ if (className.Contains("."))
+ {
+ // First, try className == FQN
+ Type result = Type.GetType(className);
+ if (result == null)
+ {
+ // Second, retry lookup after prepending the Lucene analysis package prefix
+ result = Type.GetType(LUCENE_ANALYSIS_PACKAGE_PREFIX + className);
+
+ if (result == null)
+ {
+ throw new TypeLoadException("Can't find class '" + className
+ + "' or '" + LUCENE_ANALYSIS_PACKAGE_PREFIX + className + "'");
+ }
+ }
+ return result;
+ }
+ // No dot - use analysis SPI lookup
+ string analysisComponentName = ANALYSIS_COMPONENT_SUFFIX_PATTERN.Replace(className, "", 1);
+ if (typeof(CharFilterFactory).IsAssignableFrom(expectedType))
+ {
+ return CharFilterFactory.LookupClass(analysisComponentName);
+ }
+ else if (typeof(TokenizerFactory).IsAssignableFrom(expectedType))
+ {
+ return TokenizerFactory.LookupClass(analysisComponentName);
+ }
+ else if (typeof(TokenFilterFactory).IsAssignableFrom(expectedType))
+ {
+ return TokenFilterFactory.LookupClass(analysisComponentName);
+ }
+
+ throw new TypeLoadException("Can't find class '" + className + "'");
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+
+ /// <summary>Returns the current line in the algorithm file</summary>
+ public virtual int GetLineNumber(StreamTokenizer stok)
+ {
+ return AlgLineNum + stok.LineNumber;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs
new file mode 100644
index 0000000..a20160c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs
@@ -0,0 +1,32 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Abstract class for benchmarking highlighting performance
+ /// </summary>
+ public abstract class BenchmarkHighlighter
+ {
+ public abstract int DoHighlight(IndexReader reader, int doc, string field,
+ Document document, Analyzer analyzer, string text);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs
new file mode 100644
index 0000000..16a1859
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs
@@ -0,0 +1,44 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Clear statistics data.
+ /// <para/>
+ /// Other side effects: None.
+ /// </summary>
+ public class ClearStatsTask : PerfTask
+ {
+ public ClearStatsTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.Points.ClearData();
+ return 0;
+ }
+
+ /// <seealso cref="PerfTask.ShouldNotRecordStats"/>
+ protected override bool ShouldNotRecordStats
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs
new file mode 100644
index 0000000..30e31d9
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs
@@ -0,0 +1,67 @@
+using Lucene.Net.Index;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Close index writer.
+ /// <para/>
+ /// Other side effects: index writer object in perfRunData is nullified.
+ /// <para/>
+ /// Takes optional param "doWait": if false, then close(false) is called.
+ /// </summary>
+ public class CloseIndexTask : PerfTask
+ {
+ public CloseIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ bool doWait = true;
+
+ public override int DoLogic()
+ {
+ IndexWriter iw = RunData.IndexWriter;
+ if (iw != null)
+ {
+ // If infoStream was set to output to a file, close it.
+ InfoStream infoStream = iw.Config.InfoStream;
+ if (infoStream != null)
+ {
+ infoStream.Dispose();
+ }
+ iw.Dispose(doWait);
+ RunData.IndexWriter = null;
+ }
+ return 1;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ doWait = bool.Parse(@params);
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs
new file mode 100644
index 0000000..7a8c61c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs
@@ -0,0 +1,49 @@
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Close index reader.
+ /// <para/>
+ /// Other side effects: index reader in perfRunData is nullified.
+ /// <para/>
+ /// This would cause read related tasks to reopen their own reader.
+ /// </summary>
+ public class CloseReaderTask : PerfTask
+ {
+ public CloseReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ IndexReader reader = RunData.GetIndexReader();
+ RunData.SetIndexReader(null);
+ if (reader.RefCount != 1)
+ {
+ SystemConsole.WriteLine("WARNING: CloseReader: reference count is currently " + reader.RefCount);
+ }
+ reader.DecRef();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs
new file mode 100644
index 0000000..7d94a9a
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs
@@ -0,0 +1,42 @@
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Close taxonomy index.
+ /// <para/>
+ /// Other side effects: taxonomy writer object in perfRunData is nullified.
+ /// </summary>
+ public class CloseTaxonomyIndexTask : PerfTask
+ {
+ public CloseTaxonomyIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ IOUtils.Dispose(RunData.TaxonomyWriter);
+ RunData.TaxonomyWriter = null;
+
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs
new file mode 100644
index 0000000..fc1ab27
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs
@@ -0,0 +1,47 @@
+using Lucene.Net.Facet.Taxonomy;
+using Lucene.Net.Support;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Close taxonomy reader.
+ /// <para/>
+ /// Other side effects: taxonomy reader in perfRunData is nullified.
+ /// </summary>
+ public class CloseTaxonomyReaderTask : PerfTask
+ {
+ public CloseTaxonomyReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ TaxonomyReader taxoReader = RunData.GetTaxonomyReader();
+ RunData.SetTaxonomyReader(null);
+ if (taxoReader.RefCount != 1)
+ {
+ SystemConsole.WriteLine("WARNING: CloseTaxonomyReader: reference count is currently " + taxoReader.RefCount);
+ }
+ taxoReader.Dispose();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs
new file mode 100644
index 0000000..b914371
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs
@@ -0,0 +1,62 @@
+using Lucene.Net.Index;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Commits the <see cref="IndexWriter"/>.
+ /// </summary>
+ public class CommitIndexTask : PerfTask
+ {
+ IDictionary<string, string> commitUserData;
+
+ public CommitIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ commitUserData = new Dictionary<string, string>();
+ commitUserData[OpenReaderTask.USER_DATA] = @params;
+ }
+
+ public override int DoLogic()
+ {
+ IndexWriter iw = RunData.IndexWriter;
+ if (iw != null)
+ {
+ if (commitUserData != null)
+ {
+ iw.SetCommitData(commitUserData);
+ }
+ iw.Commit();
+ }
+
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs
new file mode 100644
index 0000000..4b8cc2c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs
@@ -0,0 +1,48 @@
+using Lucene.Net.Facet.Taxonomy;
+using System;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Commits the Taxonomy Index.
+ /// </summary>
+ public class CommitTaxonomyIndexTask : PerfTask
+ {
+ public CommitTaxonomyIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ ITaxonomyWriter taxonomyWriter = RunData.TaxonomyWriter;
+ if (taxonomyWriter != null)
+ {
+ taxonomyWriter.Commit();
+ }
+ else
+ {
+ throw new InvalidOperationException("TaxonomyWriter is not currently open");
+ }
+
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs
new file mode 100644
index 0000000..36a5a14
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs
@@ -0,0 +1,48 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Consumes a <see cref="Feeds.ContentSource"/>.
+ /// </summary>
+ public class ConsumeContentSourceTask : PerfTask
+ {
+ private readonly ContentSource source;
+ private ThreadLocal<DocData> dd = new ThreadLocal<DocData>();
+
+ public ConsumeContentSourceTask(PerfRunData runData)
+ : base(runData)
+ {
+ source = runData.ContentSource;
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return "read " + recsCount + " documents from the content source";
+ }
+
+ public override int DoLogic()
+ {
+ dd.Value = source.GetNextDocData(dd.Value);
+ return 1;
+ }
+ }
+}
[08/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
new file mode 100644
index 0000000..046ed25
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateIndexTask.cs
@@ -0,0 +1,225 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Codecs;
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Creates an index.
+ /// </summary>
+ /// <remarks>
+ /// Other side effects: index writer object in perfRunData is set.
+ /// <para/>
+ /// Relevant properties:
+ /// <list type="bullet">
+ /// <item><term>merge.factor</term><description>(default 10)</description></item>
+ /// <item><term>max.buffered</term><description>(default no flush)</description></item>
+ /// <item><term>compound</term><description>(default true)</description></item>
+ /// <item><term>ram.flush.mb</term><description>[default 0]</description></item>
+ /// <item><term>merge.policy</term><description>(default Lucene.Net.Index.LogByteSizeMergePolicy, Lucene.Net)</description></item>
+ /// <item><term>merge.scheduler</term><description>(default Lucene.Net.Index.ConcurrentMergeScheduler, Lucene.Net)</description></item>
+ /// <item><term>concurrent.merge.scheduler.max.thread.count</term><description>(defaults per ConcurrentMergeScheduler)</description></item>
+ /// <item><term>concurrent.merge.scheduler.max.merge.count</term><description>(defaults per ConcurrentMergeScheduler)</description></item>
+ /// <item><term>default.codec</term><description></description></item>
+ /// </list>
+ /// <para/>
+ /// This task also supports a "writer.info.stream" property with the following
+ /// values:
+ /// <list type="bullet">
+ /// <item><term>SystemOut</term><description>Sets <see cref="IndexWriterConfig.SetInfoStream(InfoStream)"/> to <see cref="SystemConsole.Out"/>.</description></item>
+ /// <item><term>SystemErr</term><description>Sets <see cref="IndexWriterConfig.SetInfoStream(InfoStream)"/> to <see cref="SystemConsole.Error"/></description></item>
+ /// <item><term><file_name></term><description>
+ /// Attempts to create a file given that name and sets <see cref="IndexWriterConfig.SetInfoStream(InfoStream)"/>
+ /// to that file. If this denotes an invalid file name, or some error occurs, an exception will be thrown.
+ /// </description></item>
+ /// </list>
+ /// </remarks>
+ public class CreateIndexTask : PerfTask
+ {
+ public CreateIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public static IndexDeletionPolicy GetIndexDeletionPolicy(Config config)
+ {
+ string deletionPolicyName = config.Get("deletion.policy", "Lucene.Net.Index.KeepOnlyLastCommitDeletionPolicy, Lucene.Net");
+ Type deletionPolicyType = Type.GetType(deletionPolicyName);
+ if (deletionPolicyType == null)
+ {
+ throw new Exception("Unrecognized deletion policy type '" + deletionPolicyName + "'");
+ }
+ else if (deletionPolicyType.Equals(typeof(NoDeletionPolicy)))
+ {
+ return NoDeletionPolicy.INSTANCE;
+ }
+ else
+ {
+ try
+ {
+ return (IndexDeletionPolicy)Activator.CreateInstance(deletionPolicyType);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("unable to instantiate class '" + deletionPolicyName + "' as IndexDeletionPolicy", e);
+ }
+ }
+ }
+
+ public override int DoLogic()
+ {
+ PerfRunData runData = RunData;
+ Config config = runData.Config;
+ runData.IndexWriter = ConfigureWriter(config, runData, OpenMode.CREATE, null);
+ return 1;
+ }
+
+ public static IndexWriterConfig CreateWriterConfig(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit)
+ {
+ // :Post-Release-Update-Version.LUCENE_XY:
+ LuceneVersion version = (LuceneVersion)Enum.Parse(typeof(LuceneVersion), config.Get("writer.version", LuceneVersion.LUCENE_48.ToString()));
+ IndexWriterConfig iwConf = new IndexWriterConfig(version, runData.Analyzer);
+ iwConf.OpenMode = mode;
+ IndexDeletionPolicy indexDeletionPolicy = GetIndexDeletionPolicy(config);
+ iwConf.IndexDeletionPolicy = indexDeletionPolicy;
+ if (commit != null)
+ iwConf.IndexCommit = commit;
+
+
+ string mergeScheduler = config.Get("merge.scheduler",
+ "Lucene.Net.Index.ConcurrentMergeScheduler, Lucene.Net");
+ Type mergeSchedulerType = Type.GetType(mergeScheduler);
+ if (mergeSchedulerType == null)
+ {
+ throw new Exception("Unrecognized merge scheduler type '" + mergeScheduler + "'");
+ }
+ else if (mergeSchedulerType.Equals(typeof(NoMergeScheduler)))
+ {
+ iwConf.MergeScheduler = NoMergeScheduler.INSTANCE;
+ }
+ else
+ {
+ try
+ {
+ iwConf.MergeScheduler = (IMergeScheduler)Activator.CreateInstance(mergeSchedulerType);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("unable to instantiate class '" + mergeScheduler + "' as merge scheduler", e);
+ }
+
+ if (mergeScheduler.Equals("Lucene.Net.Index.ConcurrentMergeScheduler"))
+ {
+ ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)iwConf.MergeScheduler;
+ int maxThreadCount = config.Get("concurrent.merge.scheduler.max.thread.count", ConcurrentMergeScheduler.DEFAULT_MAX_THREAD_COUNT);
+ int maxMergeCount = config.Get("concurrent.merge.scheduler.max.merge.count", ConcurrentMergeScheduler.DEFAULT_MAX_MERGE_COUNT);
+ cms.SetMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+ }
+ }
+
+ string defaultCodec = config.Get("default.codec", null);
+ if (defaultCodec != null)
+ {
+ try
+ {
+ Type clazz = Type.GetType(defaultCodec);
+ iwConf.Codec = (Codec)Activator.CreateInstance(clazz);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("Couldn't instantiate Codec: " + defaultCodec, e);
+ }
+ }
+
+ string mergePolicy = config.Get("merge.policy",
+ "Lucene.Net.Index.LogByteSizeMergePolicy, Lucene.Net");
+ bool isCompound = config.Get("compound", true);
+ Type mergePolicyType = Type.GetType(mergePolicy);
+ if (mergePolicyType == null)
+ {
+ throw new Exception("Unrecognized merge policy type '" + mergePolicy + "'");
+ }
+ else if (mergePolicyType.Equals(typeof(NoMergePolicy)))
+ {
+ iwConf.MergePolicy = isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES;
+ }
+ else
+ {
+ try
+ {
+ iwConf.MergePolicy = (MergePolicy)Activator.CreateInstance(mergePolicyType);
+ }
+ catch (Exception e)
+ {
+ throw new Exception("unable to instantiate class '" + mergePolicy + "' as merge policy", e);
+ }
+ iwConf.MergePolicy.NoCFSRatio = isCompound ? 1.0 : 0.0;
+ if (iwConf.MergePolicy is LogMergePolicy)
+ {
+ LogMergePolicy logMergePolicy = (LogMergePolicy)iwConf.MergePolicy;
+ logMergePolicy.MergeFactor = config.Get("merge.factor", OpenIndexTask.DEFAULT_MERGE_PFACTOR);
+ }
+ }
+ double ramBuffer = config.Get("ram.flush.mb", OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
+ int maxBuffered = config.Get("max.buffered", OpenIndexTask.DEFAULT_MAX_BUFFERED);
+ if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH)
+ {
+ iwConf.RAMBufferSizeMB = ramBuffer;
+ iwConf.MaxBufferedDocs = maxBuffered;
+ }
+ else
+ {
+ iwConf.MaxBufferedDocs = maxBuffered;
+ iwConf.RAMBufferSizeMB = ramBuffer;
+ }
+
+ return iwConf;
+ }
+
+ public static IndexWriter ConfigureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit)
+ {
+ IndexWriterConfig iwc = CreateWriterConfig(config, runData, mode, commit);
+ string infoStreamVal = config.Get("writer.info.stream", null);
+ if (infoStreamVal != null)
+ {
+ if (infoStreamVal.Equals("SystemOut", StringComparison.Ordinal))
+ {
+ iwc.SetInfoStream(SystemConsole.Out);
+ }
+ else if (infoStreamVal.Equals("SystemErr", StringComparison.Ordinal))
+ {
+ iwc.SetInfoStream(SystemConsole.Error);
+ }
+ else
+ {
+ FileInfo f = new FileInfo(infoStreamVal);
+ iwc.SetInfoStream(new StreamWriter(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), Encoding.GetEncoding(0)));
+ }
+ }
+ IndexWriter writer = new IndexWriter(runData.Directory, iwc);
+ return writer;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateTaxonomyIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateTaxonomyIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateTaxonomyIndexTask.cs
new file mode 100644
index 0000000..15ec2ee
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CreateTaxonomyIndexTask.cs
@@ -0,0 +1,42 @@
+using Lucene.Net.Facet.Taxonomy.Directory;
+using Lucene.Net.Index;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Create a taxonomy index.
+ /// <para/>
+ /// Other side effects: taxonomy writer object in perfRunData is set.
+ /// </summary>
+ public class CreateTaxonomyIndexTask : PerfTask
+ {
+ public CreateTaxonomyIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ PerfRunData runData = RunData;
+ runData.TaxonomyWriter = new DirectoryTaxonomyWriter(runData.TaxonomyDir, OpenMode.CREATE);
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ForceMergeTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ForceMergeTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ForceMergeTask.cs
new file mode 100644
index 0000000..3e29e4d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ForceMergeTask.cs
@@ -0,0 +1,61 @@
+using Lucene.Net.Index;
+using System;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Runs forceMerge on the index.
+ /// <para/>
+ /// Other side effects: none.
+ /// </summary>
+ public class ForceMergeTask : PerfTask
+ {
+ public ForceMergeTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ int maxNumSegments = -1;
+
+ public override int DoLogic()
+ {
+ if (maxNumSegments == -1)
+ {
+ throw new InvalidOperationException("required argument (maxNumSegments) was not specified");
+ }
+ IndexWriter iw = RunData.IndexWriter;
+ iw.ForceMerge(maxNumSegments);
+ //System.out.println("forceMerge called");
+ return 1;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ maxNumSegments = (int)double.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/NearRealtimeReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/NearRealtimeReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/NearRealtimeReaderTask.cs
new file mode 100644
index 0000000..411f285
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/NearRealtimeReaderTask.cs
@@ -0,0 +1,132 @@
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Globalization;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Spawns a BG thread that periodically (defaults to 3.0
+ /// seconds, but accepts param in seconds) wakes up and asks
+ /// IndexWriter for a near real-time reader. Then runs a
+ /// single query (body: 1) sorted by docdate, and prints
+ /// time to reopen and time to run the search.
+ /// <para/>
+ /// @lucene.experimental It's also not generally usable, eg
+ /// you cannot change which query is executed.
+ /// </summary>
+ public class NearRealtimeReaderTask : PerfTask
+ {
+ internal long pauseMSec = 3000L;
+
+ internal int reopenCount;
+ internal int[] reopenTimes = new int[1];
+
+ public NearRealtimeReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ PerfRunData runData = RunData;
+
+ // Get initial reader
+ IndexWriter w = runData.IndexWriter;
+ if (w == null)
+ {
+ throw new Exception("please open the writer before invoking NearRealtimeReader");
+ }
+
+ if (runData.GetIndexReader() != null)
+ {
+ throw new Exception("please close the existing reader before invoking NearRealtimeReader");
+ }
+
+
+ long t = Support.Time.CurrentTimeMilliseconds();
+ DirectoryReader r = DirectoryReader.Open(w, true);
+ runData.SetIndexReader(r);
+ // Transfer our reference to runData
+ r.DecRef();
+
+ // TODO: gather basic metrics for reporting -- eg mean,
+ // stddev, min/max reopen latencies
+
+ // Parent sequence sets stopNow
+ reopenCount = 0;
+ while (!Stop)
+ {
+ long waitForMsec = (pauseMSec - (Support.Time.CurrentTimeMilliseconds() - t));
+ if (waitForMsec > 0)
+ {
+ Thread.Sleep((int)waitForMsec);
+ //System.out.println("NRT wait: " + waitForMsec + " msec");
+ }
+
+ t = Support.Time.CurrentTimeMilliseconds();
+ DirectoryReader newReader = DirectoryReader.OpenIfChanged(r);
+ if (newReader != null)
+ {
+ int delay = (int)(Support.Time.CurrentTimeMilliseconds() - t);
+ if (reopenTimes.Length == reopenCount)
+ {
+ reopenTimes = ArrayUtil.Grow(reopenTimes, 1 + reopenCount);
+ }
+ reopenTimes[reopenCount++] = delay;
+ // TODO: somehow we need to enable warming, here
+ runData.SetIndexReader(newReader);
+ // Transfer our reference to runData
+ newReader.DecRef();
+ r = newReader;
+ }
+ }
+ Stop = false;
+
+ return reopenCount;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ pauseMSec = (long)(1000.0 * float.Parse(@params, CultureInfo.InvariantCulture));
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ SystemConsole.WriteLine("NRT reopen times:");
+ for (int i = 0; i < reopenCount; i++)
+ {
+ SystemConsole.Write(" " + reopenTimes[i]);
+ }
+ SystemConsole.WriteLine();
+ }
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/NewAnalyzerTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/NewAnalyzerTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewAnalyzerTask.cs
new file mode 100644
index 0000000..aae5abb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewAnalyzerTask.cs
@@ -0,0 +1,189 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support.IO;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Create a new <see cref="Analyzer"/> and set it it in the getRunData() for use by all future tasks.
+ /// </summary>
+ public class NewAnalyzerTask : PerfTask
+ {
+ private IList<string> analyzerNames;
+ private int current;
+
+ public NewAnalyzerTask(PerfRunData runData)
+ : base(runData)
+ {
+ analyzerNames = new List<string>();
+ }
+
+ public static Analyzer CreateAnalyzer(string className)
+ {
+ Type clazz = Type.GetType(className);
+ try
+ {
+ // first try to use a ctor with version parameter (needed for many new Analyzers that have no default one anymore
+ return (Analyzer)Activator.CreateInstance(clazz,
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT);
+#pragma warning restore 612, 618
+ }
+ catch (MissingMethodException /*nsme*/)
+ {
+ // otherwise use default ctor
+ return (Analyzer)Activator.CreateInstance(clazz);
+ }
+ }
+
+ public override int DoLogic()
+ {
+ string analyzerName = null;
+ try
+ {
+ if (current >= analyzerNames.Count)
+ {
+ current = 0;
+ }
+ analyzerName = analyzerNames[current++];
+ Analyzer analyzer = null;
+ if (null == analyzerName || 0 == analyzerName.Length)
+ {
+ analyzerName = "Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net.Analysis.Common";
+ }
+ // First, lookup analyzerName as a named analyzer factory
+ AnalyzerFactory factory;
+ if (RunData.AnalyzerFactories.TryGetValue(analyzerName, out factory) && null != factory)
+ {
+ analyzer = factory.Create();
+ }
+ else
+ {
+ if (analyzerName.Contains("."))
+ {
+ if (analyzerName.StartsWith("Standard.", StringComparison.Ordinal))
+ {
+ analyzerName = "Lucene.Net.Analysis." + analyzerName;
+ }
+ analyzer = CreateAnalyzer(analyzerName);
+ }
+ else
+ { // No package
+ try
+ {
+ // Attempt to instantiate a core analyzer
+ string coreClassName = "Lucene.Net.Analysis.Core." + analyzerName;
+ analyzer = CreateAnalyzer(coreClassName);
+ analyzerName = coreClassName;
+ }
+ catch (TypeLoadException /*e*/)
+ {
+ // If not a core analyzer, try the base analysis package
+ analyzerName = "Lucene.Net.Analysis." + analyzerName;
+ analyzer = CreateAnalyzer(analyzerName);
+ }
+ }
+ }
+ RunData.Analyzer = analyzer;
+ }
+ catch (Exception e)
+ {
+ throw new Exception("Error creating Analyzer: " + analyzerName, e);
+ }
+ return 1;
+ }
+
+ /// <summary>
+ /// Set the params (analyzerName only), Comma-separate list of Analyzer class names. If the Analyzer lives in
+ /// Lucene.Net.Analysis, the name can be shortened by dropping the Lucene.Net.Analysis part of the Fully Qualified Class Name.
+ /// <para/>
+ /// Analyzer names may also refer to previously defined AnalyzerFactory's.
+ /// <para/>
+ /// Example Declaration:
+ /// <code>
+ /// {"NewAnalyzer" NewAnalyzer(WhitespaceAnalyzer, SimpleAnalyzer, StopAnalyzer, Standard.StandardAnalyzer) >
+ /// </code>
+ /// <para/>
+ /// Example AnalyzerFactory usage:
+ /// <code>
+ /// -AnalyzerFactory(name:'whitespace tokenized',WhitespaceTokenizer)
+ /// -NewAnalyzer('whitespace tokenized')
+ /// </code>
+ /// </summary>
+ /// <param name="params">analyzerClassName, or empty for the StandardAnalyzer</param>
+ public override void SetParams(string @params)
+ {
+
+ base.SetParams(@params);
+ StreamTokenizer stok = new StreamTokenizer(new StringReader(@params));
+ stok.QuoteChar('"');
+ stok.QuoteChar('\'');
+ stok.IsEOLSignificant = false;
+ stok.OrdinaryChar(',');
+ try
+ {
+ while (stok.NextToken() != StreamTokenizer.TT_EOF)
+ {
+ switch (stok.TokenType)
+ {
+ case ',':
+ {
+ // Do nothing
+ break;
+ }
+ case '\'':
+ case '\"':
+ case StreamTokenizer.TT_WORD:
+ {
+ analyzerNames.Add(stok.StringValue);
+ break;
+ }
+ default:
+ {
+ //throw new RuntimeException("Unexpected token: " + stok.ToString());
+ throw new Exception("Unexpected token: " + stok.ToString());
+ }
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ if (e.Message.StartsWith("Line #", StringComparison.Ordinal))
+ {
+ throw e;
+ }
+ else
+ {
+ throw new Exception("Line #" + (stok.LineNumber + AlgLineNum) + ": ", e);
+ }
+ }
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs
new file mode 100644
index 0000000..4ff00c1
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs
@@ -0,0 +1,149 @@
+using Icu.Collation;
+using Lucene.Net.Analysis;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// LUCENENET specific extension methods for the <see cref="NewCollationAnalyzerTask.Implementation"/> enumeration.
+ /// </summary>
+ public static class ImplementationExtensions
+ {
+ public static Type GetAnalyzerType(this NewCollationAnalyzerTask.Implementation impl)
+ {
+ switch (impl)
+ {
+ //case NewCollationAnalyzerTask.Implementation.JDK:
+ // return typeof(Lucene.Net.Collation.CollationKeyAnalyzer);
+
+ case NewCollationAnalyzerTask.Implementation.ICU:
+ return typeof(Lucene.Net.Collation.ICUCollationKeyAnalyzer);
+ default:
+ return typeof(Lucene.Net.Collation.ICUCollationKeyAnalyzer);
+ }
+ }
+
+ public static Type GetCollatorType(this NewCollationAnalyzerTask.Implementation impl)
+ {
+ switch (impl)
+ {
+ //case NewCollationAnalyzerTask.Implementation.JDK:
+ // return typeof(Icu.Collation.Collator);
+
+ case NewCollationAnalyzerTask.Implementation.ICU:
+ return typeof(Icu.Collation.Collator);
+ default:
+ return typeof(Icu.Collation.Collator);
+ }
+ }
+ }
+
+ public class NewCollationAnalyzerTask : PerfTask
+ {
+ /// <summary>
+ /// Different Collation implementations: currently
+ /// limited to what is provided in ICU.
+ /// <para/>
+ /// See <a href="http://site.icu-project.org/charts/collation-icu4j-sun">Comparison of implementations</a>
+ /// </summary>
+ public enum Implementation
+ {
+ //JDK, // LUCENENET: Not supported
+ ICU
+ }
+
+ private Implementation impl = Implementation.ICU; //Implementation.JDK;
+
+ public NewCollationAnalyzerTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ internal static Analyzer CreateAnalyzer(CultureInfo locale, Implementation impl)
+ {
+ // LUCENENET specific - senseless to use reflection here because we only have one
+ // collator.
+ object collator = Collator.Create(locale, Collator.Fallback.FallbackAllowed);
+
+ Type clazz = impl.GetAnalyzerType();
+ return (Analyzer)Activator.CreateInstance(clazz,
+#pragma warning disable 612, 618
+ LuceneVersion.LUCENE_CURRENT,
+#pragma warning restore 612, 618
+ collator);
+ }
+
+ public override int DoLogic()
+ {
+ try
+ {
+ CultureInfo locale = RunData.Locale;
+ if (locale == null) throw new Exception(
+ "Locale must be set with the NewLocale task!");
+ Analyzer analyzer = CreateAnalyzer(locale, impl);
+ RunData.Analyzer = analyzer;
+ SystemConsole.WriteLine("Changed Analyzer to: "
+ + analyzer.GetType().Name + "(" + locale + ")");
+ }
+ catch (Exception e)
+ {
+ throw new Exception("Error creating Analyzer: impl=" + impl, e);
+ }
+ return 1;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+
+ StringTokenizer st = new StringTokenizer(@params, ",");
+ while (st.HasMoreTokens())
+ {
+ string param = st.NextToken();
+ StringTokenizer expr = new StringTokenizer(param, ":");
+ string key = expr.NextToken();
+ string value = expr.NextToken();
+ // for now we only support the "impl" parameter.
+ // TODO: add strength, decomposition, etc
+ if (key.Equals("impl", StringComparison.Ordinal))
+ {
+ if (value.Equals("icu", StringComparison.OrdinalIgnoreCase))
+ impl = Implementation.ICU;
+ //else if (value.Equals("jdk", StringComparison.OrdinalIgnoreCase))
+ // impl = Implementation.JDK;
+ else
+ throw new Exception("Unknown parameter " + param);
+ }
+ else
+ {
+ throw new Exception("Unknown parameter " + param);
+ }
+ }
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/NewLocaleTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/NewLocaleTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewLocaleTask.cs
new file mode 100644
index 0000000..135d203
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewLocaleTask.cs
@@ -0,0 +1,97 @@
+using Lucene.Net.Support;
+using System;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Set a <see cref="CultureInfo"/> for use in benchmarking.
+ /// </summary>
+ /// <remarks>
+ /// Locales can be specified in the following ways:
+ /// <list type="bullet">
+ /// <item><description><c>de</c>: Language "de"</description></item>
+ /// <item><description><code>en,US</code>: Language "en", country "US"</description></item>
+ /// <item><description><code>no-NO</code>: Language "no", country "NO"</description></item>
+ /// <item><description><code>ROOT</code>: The <see cref="CultureInfo.InvariantCulture"/></description></item>
+ /// </list>
+ /// </remarks>
+ public class NewLocaleTask : PerfTask
+ {
+ private string culture;
+ //private string language;
+ //private string country;
+ //private string variant;
+
+ /// <summary>
+ /// Create a new <see cref="CultureInfo"/> and set it it in the RunData for
+ /// use by all future tasks.
+ /// </summary>
+ /// <param name="runData"></param>
+ public NewLocaleTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ internal static CultureInfo CreateLocale(string culture /*String language, String country, String variant*/)
+ {
+ if (culture == null || culture.Length == 0)
+ return null;
+
+ string lang = culture;
+ if (lang.Equals("ROOT", StringComparison.OrdinalIgnoreCase))
+ return CultureInfo.InvariantCulture; // Default culture
+ //lang = ""; // empty language is the root locale in the JDK
+
+ return new CultureInfo(lang);
+ }
+
+ public override int DoLogic()
+ {
+ CultureInfo locale = CreateLocale(culture /*language, country, variant*/);
+ RunData.Locale = locale;
+ SystemConsole.WriteLine("Changed Locale to: " +
+ (locale == null ? "null" :
+ (locale.EnglishName.Length == 0) ? "root locale" : locale.ToString()));
+ return 1;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ //language = country = variant = "";
+ culture = "";
+ string ignore;
+ StringTokenizer st = new StringTokenizer(@params, ",");
+ if (st.HasMoreTokens())
+ //language = st.nextToken();
+ culture = st.NextToken();
+ if (st.HasMoreTokens())
+ culture += "-" + st.NextToken();
+ if (st.HasMoreTokens())
+ ignore = st.NextToken();
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/NewRoundTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/NewRoundTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewRoundTask.cs
new file mode 100644
index 0000000..66fc685
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewRoundTask.cs
@@ -0,0 +1,44 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Increment the counter for properties maintained by Round Number.
+ /// <para/>
+ /// Other side effects: if there are props by round number, log value change.
+ /// </summary>
+ public class NewRoundTask : PerfTask
+ {
+ public NewRoundTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.Config.NewRound();
+ return 0;
+ }
+
+ /// <seealso cref="PerfTask.ShouldNotRecordStats"/>
+ protected override bool ShouldNotRecordStats
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenIndexTask.cs
new file mode 100644
index 0000000..73ec96a
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenIndexTask.cs
@@ -0,0 +1,88 @@
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Index;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Open an index writer.
+ /// </summary>
+ /// <remarks>
+ /// Other side effects: index writer object in perfRunData is set.
+ /// <para/>
+ /// Relevant properties:
+ /// <list type="bullet">
+ /// <item><term>merge.factor</term><description></description></item>
+ /// <item><term>max.buffered</term><description></description></item>
+ /// <item><term>max.field.length</term><description></description></item>
+ /// <item><term>ram.flush.mb</term><description>[default 0]</description></item>
+ /// </list>
+ /// <para/>
+ /// Accepts a param specifying the commit point as
+ /// previously saved with <see cref="CommitIndexTask"/>. If you specify
+ /// this, it rolls the index back to that commit on opening
+ /// the <see cref="IndexWriter"/>.
+ /// </remarks>
+ public class OpenIndexTask : PerfTask
+ {
+ public static readonly int DEFAULT_MAX_BUFFERED = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS;
+ public static readonly int DEFAULT_MERGE_PFACTOR = LogMergePolicy.DEFAULT_MERGE_FACTOR;
+ public static readonly double DEFAULT_RAM_FLUSH_MB = (int)IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
+ private string commitUserData;
+
+ public OpenIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ PerfRunData runData = RunData;
+ Config config = runData.Config;
+ IndexCommit ic;
+ if (commitUserData != null)
+ {
+ ic = OpenReaderTask.FindIndexCommit(runData.Directory, commitUserData);
+ }
+ else
+ {
+ ic = null;
+ }
+
+ IndexWriter writer = CreateIndexTask.ConfigureWriter(config, runData, OpenMode.APPEND, ic);
+ runData.IndexWriter = writer;
+ return 1;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ if (@params != null)
+ {
+ // specifies which commit point to open
+ commitUserData = @params;
+ }
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenReaderTask.cs
new file mode 100644
index 0000000..81adf4c
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenReaderTask.cs
@@ -0,0 +1,100 @@
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Open an index reader.
+ /// <para/>
+ /// Other side effects: index reader object in perfRunData is set.
+ /// <para/>
+ /// Optional params commitUserData eg. OpenReader(false,commit1)
+ /// </summary>
+ public class OpenReaderTask : PerfTask
+ {
+ public static readonly string USER_DATA = "userData";
+ private string commitUserData = null;
+
+ public OpenReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ Store.Directory dir = RunData.Directory;
+ DirectoryReader r = null;
+ if (commitUserData != null)
+ {
+ r = DirectoryReader.Open(OpenReaderTask.FindIndexCommit(dir, commitUserData));
+ }
+ else
+ {
+ r = DirectoryReader.Open(dir);
+ }
+ RunData.SetIndexReader(r);
+ // We transfer reference to the run data
+ r.DecRef();
+ return 1;
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ if (@params != null)
+ {
+ string[] split = @params.Split(new char[] { ',' }).TrimEnd();
+ if (split.Length > 0)
+ {
+ commitUserData = split[0];
+ }
+ }
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+
+ public static IndexCommit FindIndexCommit(Store.Directory dir, string userData)
+ {
+ IList<IndexCommit> commits = DirectoryReader.ListCommits(dir);
+ foreach (IndexCommit ic in commits)
+ {
+ IDictionary<string, string> map = ic.UserData;
+ string ud = null;
+ if (map != null)
+ {
+ //ud = map.get(USER_DATA);
+ map.TryGetValue(USER_DATA, out ud);
+ }
+ if (ud != null && ud.Equals(userData, StringComparison.Ordinal))
+ {
+ return ic;
+ }
+ }
+
+ throw new IOException("index does not contain commit with userData: " + userData);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyIndexTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyIndexTask.cs
new file mode 100644
index 0000000..5a1f38b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyIndexTask.cs
@@ -0,0 +1,41 @@
+using Lucene.Net.Facet.Taxonomy.Directory;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Open a taxonomy index.
+ /// <para/>
+ /// Other side effects: taxonomy writer object in perfRunData is set.
+ /// </summary>
+ public class OpenTaxonomyIndexTask : PerfTask
+ {
+ public OpenTaxonomyIndexTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ PerfRunData runData = RunData;
+ runData.TaxonomyWriter = new DirectoryTaxonomyWriter(runData.TaxonomyDir);
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyReaderTask.cs
new file mode 100644
index 0000000..e53738f
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/OpenTaxonomyReaderTask.cs
@@ -0,0 +1,44 @@
+using Lucene.Net.Facet.Taxonomy.Directory;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Open a taxonomy index reader.
+ /// <para/>
+ /// Other side effects: taxonomy reader object in perfRunData is set.
+ /// </summary>
+ public class OpenTaxonomyReaderTask : PerfTask
+ {
+ public OpenTaxonomyReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ PerfRunData runData = RunData;
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(runData.TaxonomyDir);
+ runData.SetTaxonomyReader(taxoReader);
+ // We transfer reference to the run data
+ taxoReader.DecRef();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/PerfTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/PerfTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/PerfTask.cs
new file mode 100644
index 0000000..0ae9dac
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/PerfTask.cs
@@ -0,0 +1,380 @@
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Support;
+using System;
+using System.Diagnostics;
+using System.Globalization;
+using System.Text;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// An abstract task to be tested for performance.
+ /// </summary>
+ /// <remarks>
+ /// Every performance task extends this class, and provides its own
+ /// <see cref="DoLogic()"/> method, which performs the actual task.
+ /// <para/>
+ /// Tasks performing some work that should be measured for the task, can override
+ /// <see cref="Setup()"/> and/or <see cref="TearDown()"/> and place that work there.
+ /// <para/>
+ /// Relevant properties:
+ /// <list type="bullet">
+ /// <item><term>task.max.depth.log</term><description></description></item>
+ /// </list>
+ /// <para/>
+ /// Also supports the following logging attributes:
+ /// <list type="bullet">
+ /// <item><term>log.step</term><description>
+ /// specifies how often to log messages about the current running
+ /// task. Default is 1000 <see cref="DoLogic()"/> invocations. Set to -1 to disable
+ /// logging.
+ /// </description></item>
+ /// <item><term>log.step.[class Task Name]</term><description>
+ /// specifies the same as 'log.step', only for a
+ /// particular task name. For example, log.step.AddDoc will be applied only for
+ /// <see cref="AddDocTask"/>. It's a way to control
+ /// per task logging settings. If you want to omit logging for any other task,
+ /// include log.step=-1. The syntax is "log.step." together with the Task's
+ /// 'short' name (i.e., without the 'Task' part).
+ /// </description></item>
+ /// </list>
+ /// </remarks>
+ public abstract class PerfTask
+ {
+ internal static readonly int DEFAULT_LOG_STEP = 1000;
+
+ private PerfRunData runData;
+
+ // propeties that all tasks have
+ private string name;
+ private int depth = 0;
+ protected int m_logStep;
+ private int logStepCount = 0;
+ private int maxDepthLogStart = 0;
+ private bool disableCounting = false;
+ protected string m_params = null;
+
+ private bool runInBackground;
+ private int deltaPri;
+
+ // The first line of this task's definition in the alg file
+ private int algLineNum = 0;
+
+ protected static readonly string NEW_LINE = Environment.NewLine;
+
+ /// <summary>
+ /// Should not be used externally
+ /// </summary>
+ private PerfTask()
+ {
+ name = GetType().Name;
+ if (name.EndsWith("Task", StringComparison.Ordinal))
+ {
+ name = name.Substring(0, name.Length - 4);
+ }
+ }
+
+ public virtual void SetRunInBackground(int deltaPri)
+ {
+ runInBackground = true;
+ this.deltaPri = deltaPri;
+ }
+
+ public virtual bool RunInBackground
+ {
+ get { return runInBackground; }
+ }
+
+ public virtual int BackgroundDeltaPriority
+ {
+ get { return deltaPri; }
+ }
+
+ // LUCENENET specific - made private and
+ // added Stop property because volatile
+ // fields cannot be protected.
+ private volatile bool stopNow;
+
+ protected bool Stop
+ {
+ get { return stopNow; }
+ set { stopNow = value; }
+ }
+ public virtual void StopNow()
+ {
+ stopNow = true;
+ }
+
+ public PerfTask(PerfRunData runData)
+ : this()
+ {
+ this.runData = runData;
+ Config config = runData.Config;
+ this.maxDepthLogStart = config.Get("task.max.depth.log", 0);
+
+ string logStepAtt = "log.step";
+ string taskLogStepAtt = "log.step." + name;
+ if (config.Get(taskLogStepAtt, null) != null)
+ {
+ logStepAtt = taskLogStepAtt;
+ }
+
+ // It's important to read this from Config, to support vals-by-round.
+ m_logStep = config.Get(logStepAtt, DEFAULT_LOG_STEP);
+ // To avoid the check 'if (logStep > 0)' in tearDown(). This effectively
+ // turns logging off.
+ if (m_logStep <= 0)
+ {
+ m_logStep = int.MaxValue;
+ }
+ }
+
+ public virtual object Clone()
+ {
+ // tasks having non primitive data structures should override this.
+ // otherwise parallel running of a task sequence might not run correctly.
+ return (PerfTask)base.MemberwiseClone();
+ }
+
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ }
+
+ /// <summary>
+ /// Run the task, record statistics.
+ /// </summary>
+ /// <param name="reportStats"></param>
+ /// <returns>Number of work items done by this task.</returns>
+ public int RunAndMaybeStats(bool reportStats)
+ {
+ int count;
+ if (!reportStats || ShouldNotRecordStats)
+ {
+ Setup();
+ count = DoLogic();
+ count = disableCounting ? 0 : count;
+ TearDown();
+ return count;
+ }
+ if (reportStats && depth <= maxDepthLogStart && !ShouldNeverLogAtStart)
+ {
+ SystemConsole.WriteLine("------------> starting task: " + GetName());
+ }
+ Setup();
+ Points pnts = runData.Points;
+ TaskStats ts = pnts.MarkTaskStart(this, runData.Config.RoundNumber);
+ count = DoLogic();
+ count = disableCounting ? 0 : count;
+ pnts.MarkTaskEnd(ts, count);
+ TearDown();
+ return count;
+ }
+
+ /// <summary>
+ /// Perform the task once (ignoring repetitions specification).
+ /// Return number of work items done by this task.
+ /// For indexing that can be number of docs added.
+ /// For warming that can be number of scanned items, etc.
+ /// </summary>
+ /// <returns>Number of work items done by this task.</returns>
+ public abstract int DoLogic();
+
+ /// <summary>
+ /// Returns the name.
+ /// </summary>
+ public virtual string GetName()
+ {
+ if (m_params == null)
+ {
+ return name;
+ }
+ return new StringBuilder(name).Append('(').Append(m_params).Append(')').ToString();
+ }
+
+ /// <summary>
+ /// Sets the name.
+ /// </summary>
+ /// <param name="name">The name to set.</param>
+ protected virtual void SetName(string name)
+ {
+ this.name = name;
+ }
+
+ /// <summary>
+ /// Gets the run data.
+ /// </summary>
+ public virtual PerfRunData RunData
+ {
+ get { return runData; }
+ }
+
+ /// <summary>
+ /// Gets or Sets the depth.
+ /// </summary>
+ public virtual int Depth
+ {
+ get { return depth; }
+ set { depth = value; }
+ }
+
+ // compute a blank string padding for printing this task indented by its depth
+ internal string GetPadding()
+ {
+ char[] c = new char[4 * Depth];
+ for (int i = 0; i < c.Length; i++) c[i] = ' ';
+ return new string(c);
+ }
+
+ public override string ToString()
+ {
+ string padd = GetPadding();
+ StringBuilder sb = new StringBuilder(padd);
+ if (disableCounting)
+ {
+ sb.Append('-');
+ }
+ sb.Append(GetName());
+ if (RunInBackground)
+ {
+ sb.Append(" &");
+ int x = BackgroundDeltaPriority;
+ if (x != 0)
+ {
+ sb.Append(x);
+ }
+ }
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Returns the maxDepthLogStart.
+ /// </summary>
+ internal int MaxDepthLogStart
+ {
+ get { return maxDepthLogStart; }
+ }
+
+ protected virtual string GetLogMessage(int recsCount)
+ {
+ return "processed " + recsCount + " records";
+ }
+
+ /// <summary>
+ /// Tasks that should never log at start can override this.
+ /// Returns <c>true</c> if this task should never log when it start.
+ /// </summary>
+ protected virtual bool ShouldNeverLogAtStart
+ {
+ get { return false; }
+ }
+
+ /// <summary>
+ /// Tasks that should not record statistics can override this.
+ /// Returns <c>true</c> if this task should never record its statistics.
+ /// </summary>
+ protected virtual bool ShouldNotRecordStats
+ {
+ get { return false; }
+ }
+
+ /// <summary>
+ /// Task setup work that should not be measured for that specific task. By
+ /// default it does nothing, but tasks can implement this, moving work from
+ /// <see cref="DoLogic()"/> to this method. Only the work done in <see cref="DoLogic()"/>
+ /// is measured for this task. Notice that higher level (sequence) tasks
+ /// containing this task would then measure larger time than the sum of their
+ /// contained tasks.
+ /// </summary>
+ public virtual void Setup()
+ {
+ }
+
+ /// <summary>
+ /// Task teardown work that should not be measured for that specific task. By
+ /// default it does nothing, but tasks can implement this, moving work from
+ /// <see cref="DoLogic()"/> to this method. Only the work done in <see cref="DoLogic()"/>
+ /// is measured for this task. Notice that higher level (sequence) tasks
+ /// containing this task would then measure larger time than the sum of their
+ /// contained tasks.
+ /// </summary>
+ public virtual void TearDown()
+ {
+ if (++logStepCount % m_logStep == 0)
+ {
+ double time = (((Stopwatch.GetTimestamp() / Stopwatch.Frequency) * 1000) - runData.StartTimeMillis) / 1000.0;
+ SystemConsole.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:0000000.00}", time) + " sec --> "
+ + Thread.CurrentThread.Name + " " + GetLogMessage(logStepCount));
+ }
+ }
+
+ /// <summary>
+ /// Sub classes that support parameters must override this method to return
+ /// <c>true</c> if this task supports command line params.
+ /// </summary>
+ public virtual bool SupportsParams
+ {
+ get { return false; }
+ }
+
+ /// <summary>
+ /// Set the params of this task.
+ /// </summary>
+ /// <exception cref="NotSupportedException">For tasks supporting command line parameters.</exception>
+ public virtual void SetParams(string @params)
+ {
+ if (!SupportsParams)
+ {
+ throw new NotSupportedException(GetName() + " does not support command line parameters.");
+ }
+ this.m_params = @params;
+ }
+
+ /// <summary>
+ /// Gets the Params.
+ /// </summary>
+ public virtual string Params
+ {
+ get { return m_params; }
+ }
+
+ /// <summary>
+ /// Return <c>true</c> if counting is disabled for this task.
+ /// </summary>
+ public virtual bool DisableCounting
+ {
+ get { return disableCounting; }
+ set { disableCounting = value; }
+ }
+
+ public virtual int AlgLineNum
+ {
+ get { return algLineNum; }
+ set { algLineNum = value; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/PrintReaderTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/PrintReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/PrintReaderTask.cs
new file mode 100644
index 0000000..3453fc5
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/PrintReaderTask.cs
@@ -0,0 +1,60 @@
+using Lucene.Net.Index;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Opens a reader and prints basic statistics.
+ /// </summary>
+ public class PrintReaderTask : PerfTask
+ {
+ private string userData = null;
+
+ public PrintReaderTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ userData = @params;
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+
+ public override int DoLogic()
+ {
+ Directory dir = RunData.Directory;
+ IndexReader r = null;
+ if (userData == null)
+ r = DirectoryReader.Open(dir);
+ else
+ r = DirectoryReader.Open(OpenReaderTask.FindIndexCommit(dir, userData));
+ SystemConsole.WriteLine("--> numDocs:" + r.NumDocs + " dels:" + r.NumDeletedDocs);
+ r.Dispose();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTask.cs
new file mode 100644
index 0000000..3eeda9b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTask.cs
@@ -0,0 +1,339 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Read index (abstract) task.
+ /// Sub classes implement <see cref="WithSearch"/>, <see cref="WithWarm"/>, <see cref="WithTraverse"/> and <see cref="WithRetrieve"/>
+ /// </summary>
+ /// <remarks>
+ /// Note: All ReadTasks reuse the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// The <c>search.num.hits</c> config parameter sets
+ /// the top number of hits to collect during searching. If
+ /// <c>print.hits.field</c> is set, then each hit is
+ /// printed along with the value of that field.
+ /// <para/>
+ /// Other side effects: none.
+ /// </remarks>
+ public abstract class ReadTask : PerfTask
+ {
+ private readonly IQueryMaker queryMaker;
+
+ public ReadTask(PerfRunData runData)
+ : base(runData)
+ {
+ if (WithSearch)
+ {
+ queryMaker = GetQueryMaker();
+ }
+ else
+ {
+ queryMaker = null;
+ }
+ }
+
+ public override int DoLogic()
+ {
+ int res = 0;
+
+ // open reader or use existing one
+ IndexSearcher searcher = RunData.GetIndexSearcher();
+
+ IndexReader reader;
+
+ bool closeSearcher;
+ if (searcher == null)
+ {
+ // open our own reader
+ Directory dir = RunData.Directory;
+ reader = DirectoryReader.Open(dir);
+ searcher = new IndexSearcher(reader);
+ closeSearcher = true;
+ }
+ else
+ {
+ // use existing one; this passes +1 ref to us
+ reader = searcher.IndexReader;
+ closeSearcher = false;
+ }
+
+ // optionally warm and add num docs traversed to count
+ if (WithWarm)
+ {
+ Document doc = null;
+ IBits liveDocs = MultiFields.GetLiveDocs(reader);
+ for (int m = 0; m < reader.MaxDoc; m++)
+ {
+ if (null == liveDocs || liveDocs.Get(m))
+ {
+ doc = reader.Document(m);
+ res += (doc == null ? 0 : 1);
+ }
+ }
+ }
+
+ if (WithSearch)
+ {
+ res++;
+ Query q = queryMaker.MakeQuery();
+ Sort sort = Sort;
+ TopDocs hits = null;
+ int numHits = NumHits;
+ if (numHits > 0)
+ {
+ if (WithCollector == false)
+ {
+ if (sort != null)
+ {
+ // TODO: instead of always passing false we
+ // should detect based on the query; if we make
+ // the IndexSearcher search methods that take
+ // Weight public again, we can go back to
+ // pulling the Weight ourselves:
+ TopFieldCollector collector = TopFieldCollector.Create(sort, numHits,
+ true, WithScore,
+ WithMaxScore,
+ false);
+ searcher.Search(q, null, collector);
+ hits = collector.GetTopDocs();
+ }
+ else
+ {
+ hits = searcher.Search(q, numHits);
+ }
+ }
+ else
+ {
+ ICollector collector = CreateCollector();
+ searcher.Search(q, null, collector);
+ //hits = collector.topDocs();
+ }
+
+ string printHitsField = RunData.Config.Get("print.hits.field", null);
+ if (hits != null && printHitsField != null && printHitsField.Length > 0)
+ {
+ SystemConsole.WriteLine("totalHits = " + hits.TotalHits);
+ SystemConsole.WriteLine("maxDoc() = " + reader.MaxDoc);
+ SystemConsole.WriteLine("numDocs() = " + reader.NumDocs);
+ for (int i = 0; i < hits.ScoreDocs.Length; i++)
+ {
+ int docID = hits.ScoreDocs[i].Doc;
+ Document doc = reader.Document(docID);
+ SystemConsole.WriteLine(" " + i + ": doc=" + docID + " score=" + hits.ScoreDocs[i].Score + " " + printHitsField + " =" + doc.Get(printHitsField));
+ }
+ }
+
+ if (WithTraverse)
+ {
+ ScoreDoc[] scoreDocs = hits.ScoreDocs;
+ int traversalSize = Math.Min(scoreDocs.Length, TraversalSize);
+
+ if (traversalSize > 0)
+ {
+ bool retrieve = WithRetrieve;
+ int numHighlight = Math.Min(NumToHighlight, scoreDocs.Length);
+ Analyzer analyzer = RunData.Analyzer;
+ BenchmarkHighlighter highlighter = null;
+ if (numHighlight > 0)
+ {
+ highlighter = GetBenchmarkHighlighter(q);
+ }
+ for (int m = 0; m < traversalSize; m++)
+ {
+ int id = scoreDocs[m].Doc;
+ res++;
+ if (retrieve)
+ {
+ Document document = RetrieveDoc(reader, id);
+ res += document != null ? 1 : 0;
+ if (numHighlight > 0 && m < numHighlight)
+ {
+ ICollection<string> fieldsToHighlight = GetFieldsToHighlight(document);
+ foreach (string field in fieldsToHighlight)
+ {
+ string text = document.Get(field);
+ res += highlighter.DoHighlight(reader, id, field, document, analyzer, text);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (closeSearcher)
+ {
+ reader.Dispose();
+ }
+ else
+ {
+ // Release our +1 ref from above
+ reader.DecRef();
+ }
+ return res;
+ }
+
+ protected virtual ICollector CreateCollector()
+ {
+ return TopScoreDocCollector.Create(NumHits, true);
+ }
+
+
+ protected virtual Document RetrieveDoc(IndexReader ir, int id)
+ {
+ return ir.Document(id);
+ }
+
+ /// <summary>
+ /// Return query maker used for this task.
+ /// </summary>
+ public abstract IQueryMaker GetQueryMaker();
+
+ /// <summary>
+ /// Return <c>true</c> if search should be performed.
+ /// </summary>
+ public abstract bool WithSearch { get; }
+
+ public virtual bool WithCollector
+ {
+ get { return false; }
+ }
+
+
+ /// <summary>
+ /// Return <c>true</c> if warming should be performed.
+ /// </summary>
+ public abstract bool WithWarm { get; }
+
+ /// <summary>
+ /// Return <c>true</c> if, with search, results should be traversed.
+ /// </summary>
+ public abstract bool WithTraverse { get; }
+
+ /// <summary>
+ /// Whether scores should be computed (only useful with
+ /// field sort)
+ /// </summary>
+ public virtual bool WithScore
+ {
+ get { return true; }
+ }
+
+ /// <summary>
+ /// Whether maxScores should be computed (only useful with
+ /// field sort)
+ /// </summary>
+ public virtual bool WithMaxScore
+ {
+ get { return true; }
+ }
+
+ /// <summary>
+ /// Specify the number of hits to traverse. Tasks should override this if they want to restrict the number
+ /// of hits that are traversed when <see cref="WithTraverse"/> is <c>true</c>. Must be greater than 0.
+ /// <para/>
+ /// Read task calculates the traversal as: <c>Math.Min(hits.Length, TraversalSize)</c>
+ /// </summary>
+ /// <remarks>
+ /// Unless overridden, the return value is <see cref="int.MaxValue"/>.
+ /// </remarks>
+ public virtual int TraversalSize
+ {
+ get { return int.MaxValue; }
+ }
+
+ internal static readonly int DEFAULT_SEARCH_NUM_HITS = 10;
+ private int numHits;
+
+ public override void Setup()
+ {
+ base.Setup();
+ numHits = RunData.Config.Get("search.num.hits", DEFAULT_SEARCH_NUM_HITS);
+ }
+
+ /// <summary>
+ /// Specify the number of hits to retrieve. Tasks should override this if they want to restrict the number
+ /// of hits that are collected during searching. Must be greater than 0.
+ /// <para/>
+ /// Returns 10 by default, or <c>search.num.hits</c> config if set.
+ /// </summary>
+ public virtual int NumHits
+ {
+ get { return numHits; }
+ }
+
+ /// <summary>
+ /// Return <c>true</c> if, with search & results traversing, docs should be retrieved.
+ /// </summary>
+ public abstract bool WithRetrieve { get; }
+
+ /// <summary>
+ /// The number of documents to highlight. 0 means no docs will be highlighted.
+ /// </summary>
+ public virtual int NumToHighlight
+ {
+ get { return 0; }
+ }
+
+ /// <summary>
+ /// Return an appropriate highlighter to be used with
+ /// highlighting tasks.
+ /// </summary>
+ /// <param name="q"></param>
+ /// <returns></returns>
+ protected virtual BenchmarkHighlighter GetBenchmarkHighlighter(Query q)
+ {
+ return null;
+ }
+
+ public virtual Sort Sort
+ {
+ get { return null; }
+ }
+
+ /// <summary>
+ /// Define the fields to highlight. Base implementation returns all fields.
+ /// </summary>
+ /// <param name="document">The <see cref="Document"/>.</param>
+ /// <returns>An <see cref="T:ICollection{string}"/> of <see cref="Field"/> names.</returns>
+ protected virtual ICollection<string> GetFieldsToHighlight(Document document)
+ {
+ IList<IIndexableField> fields = document.Fields;
+ ISet<string> result = new HashSet<string>(/*fields.Count*/);
+ foreach (IIndexableField f in fields)
+ {
+ result.Add(f.Name);
+ }
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTokensTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTokensTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTokensTask.cs
new file mode 100644
index 0000000..1a8125d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ReadTokensTask.cs
@@ -0,0 +1,160 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.TokenAttributes;
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simple task to test performance of tokenizers. It just
+ /// creates a token stream for each field of the document and
+ /// read all tokens out of that stream.
+ /// </summary>
+ public class ReadTokensTask : PerfTask
+ {
+ public ReadTokensTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ private int totalTokenCount = 0;
+
+ // volatile data passed between setup(), doLogic(), tearDown().
+ private Document doc = null;
+
+ public override void Setup()
+ {
+ base.Setup();
+ DocMaker docMaker = RunData.DocMaker;
+ doc = docMaker.MakeDocument();
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return "read " + recsCount + " docs; " + totalTokenCount + " tokens";
+ }
+
+ public override void TearDown()
+ {
+ doc = null;
+ base.TearDown();
+ }
+
+ public override int DoLogic()
+ {
+ IList<IIndexableField> fields = doc.Fields;
+ Analyzer analyzer = RunData.Analyzer;
+ int tokenCount = 0;
+ foreach (IIndexableField field in fields)
+ {
+ if (!field.FieldType.IsTokenized ||
+ field is Int32Field ||
+ field is Int64Field ||
+ field is SingleField ||
+ field is DoubleField)
+ {
+ continue;
+ }
+
+ using (TokenStream stream = field.GetTokenStream(analyzer))
+ {
+ // reset the TokenStream to the first token
+ stream.Reset();
+
+ ITermToBytesRefAttribute termAtt = stream.GetAttribute<ITermToBytesRefAttribute>();
+ while (stream.IncrementToken())
+ {
+ termAtt.FillBytesRef();
+ tokenCount++;
+ }
+ stream.End();
+ }
+ }
+ totalTokenCount += tokenCount;
+ return tokenCount;
+ }
+
+ /// <summary>
+ /// Simple StringReader that can be reset to a new string;
+ /// we use this when tokenizing the string value from a
+ /// Field.
+ /// </summary>
+ internal ReusableStringReader stringReader = new ReusableStringReader();
+
+ internal sealed class ReusableStringReader : TextReader
+ {
+ int upto;
+ int left;
+ string s;
+ internal void Init(string s)
+ {
+ this.s = s;
+ left = s.Length;
+ this.upto = 0;
+ }
+
+ public override int Read()
+ {
+ char[] result = new char[1];
+ if (Read(result, 0, 1, false) != -1)
+ {
+ return result[0];
+ }
+ return -1;
+ }
+ public override int Read(char[] c, int off, int len)
+ {
+ return Read(c, off, len, true);
+ }
+
+ private int Read(char[] c, int off, int len, bool returnZeroWhenComplete)
+ {
+ if (left > len)
+ {
+ s.CopyTo(upto, c, off, upto + len);
+ upto += len;
+ left -= len;
+ return len;
+ }
+ else if (0 == left)
+ {
+ if (returnZeroWhenComplete)
+ {
+ return 0; // .NET semantics
+ }
+ return -1;
+ }
+ else
+ {
+ s.CopyTo(upto, c, off, upto + left);
+ int r = left;
+ left = 0;
+ upto = s.Length;
+ return r;
+ }
+ }
+
+ protected override void Dispose(bool disposing) { }
+ }
+ }
+}