You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2016/04/22 16:13:16 UTC
lucenenet git commit: More work on Lucene.Net.Analysis.Common
Repository: lucenenet
Updated Branches:
refs/heads/master c006314c5 -> 8e144a0c8
More work on Lucene.Net.Analysis.Common
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/8e144a0c
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/8e144a0c
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/8e144a0c
Branch: refs/heads/master
Commit: 8e144a0c8a49a584b3fb21bb2b5db01ce82bdf87
Parents: c006314
Author: Itamar Syn-Hershko <it...@code972.com>
Authored: Fri Apr 22 07:12:57 2016 -0700
Committer: Itamar Syn-Hershko <it...@code972.com>
Committed: Fri Apr 22 07:12:57 2016 -0700
----------------------------------------------------------------------
.../Analysis/CharFilter/BaseCharFilter.cs | 4 +-
.../CharFilter/HTMLStripCharFilterFactory.cs | 4 +-
.../Analysis/CharFilter/MappingCharFilter.cs | 54 ++---
.../CharFilter/MappingCharFilterFactory.cs | 3 -
.../Analysis/CharFilter/NormalizeCharMap.cs | 210 +++++++++----------
.../Analysis/Core/LetterTokenizer.cs | 2 +-
.../Miscellaneous/HyphenatedWordsFilter.cs | 5 +-
.../Miscellaneous/LimitTokenCountAnalyzer.cs | 2 +-
.../Miscellaneous/PerFieldAnalyzerWrapper.cs | 2 +-
.../PrefixAndSuffixAwareTokenFilter.cs | 167 +++++++--------
.../Miscellaneous/SetKeywordMarkerFilter.cs | 2 -
.../Analysis/Reverse/ReverseStringFilter.cs | 28 ++-
.../Reverse/ReverseStringFilterFactory.cs | 4 +-
.../Analysis/Util/AbstractAnalysisFactory.cs | 1 -
.../Analysis/Util/CharTokenizer.cs | 4 +-
.../Lucene.Net.Analysis.Common.csproj | 22 ++
src/Lucene.Net.Core/Analysis/CharFilter.cs | 2 +-
17 files changed, 255 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/BaseCharFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/BaseCharFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/BaseCharFilter.cs
index 845a528..908759a 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/BaseCharFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/BaseCharFilter.cs
@@ -42,7 +42,7 @@ namespace Lucene.Net.Analysis.CharFilters
/// <summary>
/// Retrieve the corrected offset. </summary>
- protected internal override int Correct(int currentOff)
+ protected override int Correct(int currentOff)
{
if (offsets == null || currentOff < offsets[0])
{
@@ -118,7 +118,7 @@ namespace Lucene.Net.Analysis.CharFilters
diffs = ArrayUtil.Grow(diffs);
}
- Debug.Assert(size == 0 || off >= offsets[size - 1]) : "Offset #" + size + "(" + off + ") is less than the last recorded offset " + offsets[size - 1] + "\n" + Arrays.ToString(offsets) + "\n" + Arrays.ToString(diffs);
+ Debug.Assert(size == 0 || off >= offsets[size - 1], "Offset #" + size + "(" + off + ") is less than the last recorded offset " + offsets[size - 1] + "\n" + Arrays.ToString(offsets) + "\n" + Arrays.ToString(diffs));
if (size == 0 || off != offsets[size - 1])
{
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/HTMLStripCharFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/HTMLStripCharFilterFactory.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/HTMLStripCharFilterFactory.cs
index 019b09e..c7b3ba1 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/HTMLStripCharFilterFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/HTMLStripCharFilterFactory.cs
@@ -1,5 +1,5 @@
using System.Collections.Generic;
-using org.apache.lucene.analysis.util;
+using System.IO;
namespace Lucene.Net.Analysis.CharFilters
{
@@ -46,7 +46,7 @@ namespace Lucene.Net.Analysis.CharFilters
}
}
- public override HTMLStripCharFilter create(Reader input)
+ public override HTMLStripCharFilter create(TextReader input)
{
HTMLStripCharFilter charFilter;
if (null == escapedTags)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs
index 06ddf74..ec57c42 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilter.cs
@@ -21,7 +21,6 @@ using System.IO;
using Lucene.Net.Analysis.Util;
using Lucene.Net.Util;
using Lucene.Net.Util.Fst;
-using org.apache.lucene.analysis.charfilter;
namespace Lucene.Net.Analysis.CharFilters
{
@@ -49,7 +48,7 @@ namespace Lucene.Net.Analysis.CharFilters
private int inputOff;
/// <summary>
- /// Default constructor that takes a <seealso cref="Reader"/>. </summary>
+ /// Default constructor that takes a <seealso cref="TextReader"/>. </summary>
public MappingCharFilter(NormalizeCharMap normMap, TextReader @in) : base(@in)
{
buffer.Reset(@in);
@@ -70,21 +69,19 @@ namespace Lucene.Net.Analysis.CharFilters
public override void Reset()
{
input.Reset();
- buffer.reset(input);
+ buffer.Reset(input);
replacement = null;
inputOff = 0;
}
-//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
-//ORIGINAL LINE: @Override public int read() throws java.io.IOException
- public override int read()
+ public override int Read()
{
//System.out.println("\nread");
while (true)
{
- if (replacement != null && replacementPointer < replacement.length)
+ if (replacement != null && replacementPointer < replacement.Length)
{
//System.out.println(" return repl[" + replacementPointer + "]=" + replacement.chars[replacement.offset + replacementPointer]);
return replacement.chars[replacement.offset + replacementPointer++];
@@ -105,25 +102,23 @@ namespace Lucene.Net.Analysis.CharFilters
int lastMatchLen = -1;
CharsRef lastMatch = null;
-//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
-//ORIGINAL LINE: final int firstCH = buffer.get(inputOff);
- int firstCH = buffer.get(inputOff);
+ int firstCH = buffer.Get(inputOff);
if (firstCH != -1)
{
FST.Arc<CharsRef> arc = cachedRootArcs[Convert.ToChar((char) firstCH)];
if (arc != null)
{
- if (!FST.targetHasArcs(arc))
+ if (!FST.TargetHasArcs(arc))
{
// Fast pass for single character match:
Debug.Assert(arc.Final);
lastMatchLen = 1;
- lastMatch = arc.output;
+ lastMatch = arc.Output;
}
else
{
int lookahead = 0;
- CharsRef output = arc.output;
+ CharsRef output = arc.Output;
while (true)
{
lookahead++;
@@ -132,27 +127,27 @@ namespace Lucene.Net.Analysis.CharFilters
{
// Match! (to node is final)
lastMatchLen = lookahead;
- lastMatch = outputs.add(output, arc.nextFinalOutput);
+ lastMatch = outputs.Add(output, arc.NextFinalOutput);
// Greedy: keep searching to see if there's a
// longer match...
}
- if (!FST.targetHasArcs(arc))
+ if (!FST.TargetHasArcs(arc))
{
break;
}
- int ch = buffer.get(inputOff + lookahead);
+ int ch = buffer.Get(inputOff + lookahead);
if (ch == -1)
{
break;
}
- if ((arc = map.findTargetArc(ch, arc, scratchArc, fstReader)) == null)
+ if ((arc = map.FindTargetArc(ch, arc, scratchArc, fstReader)) == null)
{
// Dead end
break;
}
- output = outputs.add(output, arc.output);
+ output = outputs.Add(output, arc.Output);
}
}
}
@@ -162,20 +157,15 @@ namespace Lucene.Net.Analysis.CharFilters
{
inputOff += lastMatchLen;
//System.out.println(" match! len=" + lastMatchLen + " repl=" + lastMatch);
-
-//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
-//ORIGINAL LINE: final int diff = lastMatchLen - lastMatch.length;
- int diff = lastMatchLen - lastMatch.length;
+ int diff = lastMatchLen - lastMatch.Length;
if (diff != 0)
{
-//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
-//ORIGINAL LINE: final int prevCumulativeDiff = getLastCumulativeDiff();
int prevCumulativeDiff = LastCumulativeDiff;
if (diff > 0)
{
// Replacement is shorter than matched input:
- addOffCorrectMap(inputOff - diff - prevCumulativeDiff, prevCumulativeDiff + diff);
+ AddOffCorrectMap(inputOff - diff - prevCumulativeDiff, prevCumulativeDiff + diff);
}
else
{
@@ -187,7 +177,7 @@ namespace Lucene.Net.Analysis.CharFilters
int outputStart = inputOff - prevCumulativeDiff;
for (int extraIDX = 0;extraIDX < -diff;extraIDX++)
{
- addOffCorrectMap(outputStart + extraIDX, prevCumulativeDiff - extraIDX - 1);
+ AddOffCorrectMap(outputStart + extraIDX, prevCumulativeDiff - extraIDX - 1);
}
}
}
@@ -198,27 +188,23 @@ namespace Lucene.Net.Analysis.CharFilters
}
else
{
-//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
-//ORIGINAL LINE: final int ret = buffer.get(inputOff);
- int ret = buffer.get(inputOff);
+ int ret = buffer.Get(inputOff);
if (ret != -1)
{
inputOff++;
- buffer.freeBefore(inputOff);
+ buffer.FreeBefore(inputOff);
}
return ret;
}
}
}
-//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
-//ORIGINAL LINE: @Override public int read(char[] cbuf, int off, int len) throws java.io.IOException
- public override int read(char[] cbuf, int off, int len)
+ public override int Read(char[] cbuf, int off, int len)
{
int numRead = 0;
for (int i = off; i < off + len; i++)
{
- int c = read();
+ int c = Read();
if (c == -1)
{
break;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilterFactory.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilterFactory.cs
index fb9c99c..7a4ac68 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilterFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/MappingCharFilterFactory.cs
@@ -1,9 +1,6 @@
using System.Collections.Generic;
using System.IO;
-using Lucene.Net.Analysis.Util;
using Lucene.Net.Support;
-using org.apache.lucene.analysis.charfilter;
-using org.apache.lucene.analysis.util;
namespace Lucene.Net.Analysis.CharFilters
{
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs
index f26d9fd..39a9751 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/CharFilter/NormalizeCharMap.cs
@@ -25,118 +25,118 @@ namespace Lucene.Net.Analysis.CharFilters
{
// TODO: save/load?
- /// <summary>
- /// Holds a map of String input to String output, to be used
- /// with <seealso cref="Builder"/>. Use the <seealso cref="MappingCharFilter"/>
- /// to create this.
- /// </summary>
- public class NormalizeCharMap
- {
+ /// <summary>
+ /// Holds a map of String input to String output, to be used
+ /// with <seealso cref="Builder"/>. Use the <seealso cref="MappingCharFilter"/>
+ /// to create this.
+ /// </summary>
+ public class NormalizeCharMap
+ {
- internal readonly FST<CharsRef> map;
- internal readonly IDictionary<char?, FST.Arc<CharsRef>> cachedRootArcs = new Dictionary<char?, FST.Arc<CharsRef>>();
+ internal readonly FST<CharsRef> map;
+ internal readonly IDictionary<char?, FST.Arc<CharsRef>> cachedRootArcs = new Dictionary<char?, FST.Arc<CharsRef>>();
- // Use the builder to create:
- private NormalizeCharMap(FST<CharsRef> map)
- {
- this.map = map;
- if (map != null)
- {
- try
- {
- // Pre-cache root arcs:
- var scratchArc = new FST.Arc<CharsRef>();
- FST.BytesReader fstReader = map.BytesReader;
- map.GetFirstArc(scratchArc);
- if (FST<CharsRef>.TargetHasArcs(scratchArc))
- {
- map.ReadFirstRealTargetArc(scratchArc.Target, scratchArc, fstReader);
- while (true)
- {
- Debug.Assert(scratchArc.Label != FST<CharsRef>.END_LABEL); // LUCENENET TODO END_LABEL shouldn't be under generic?
- cachedRootArcs[Convert.ToChar((char) scratchArc.Label)] = (new FST.Arc<CharsRef>()).CopyFrom(scratchArc);
- if (scratchArc.Last)
- {
- break;
- }
- map.ReadNextRealArc(scratchArc, fstReader);
- }
- }
- //System.out.println("cached " + cachedRootArcs.size() + " root arcs");
- }
- catch (IOException ioe)
- {
- // Bogus FST IOExceptions!! (will never happen)
- throw new Exception(ioe);
- }
- }
- }
+ // Use the builder to create:
+ private NormalizeCharMap(FST<CharsRef> map)
+ {
+ this.map = map;
+ if (map != null)
+ {
+ try
+ {
+ // Pre-cache root arcs:
+ var scratchArc = new FST.Arc<CharsRef>();
+ FST.BytesReader fstReader = map.BytesReader;
+ map.GetFirstArc(scratchArc);
+ if (FST<CharsRef>.TargetHasArcs(scratchArc))
+ {
+ map.ReadFirstRealTargetArc(scratchArc.Target, scratchArc, fstReader);
+ while (true)
+ {
+ Debug.Assert(scratchArc.Label != FST<CharsRef>.END_LABEL); // LUCENENET TODO END_LABEL shouldn't be under generic?
+ cachedRootArcs[Convert.ToChar((char)scratchArc.Label)] = (new FST.Arc<CharsRef>()).CopyFrom(scratchArc);
+ if (scratchArc.Last)
+ {
+ break;
+ }
+ map.ReadNextRealArc(scratchArc, fstReader);
+ }
+ }
+ //System.out.println("cached " + cachedRootArcs.size() + " root arcs");
+ }
+ catch (IOException ioe)
+ {
+ // Bogus FST IOExceptions!! (will never happen)
+ throw new Exception("Should never happen", ioe);
+ }
+ }
+ }
- /// <summary>
- /// Builds an NormalizeCharMap.
- /// <para>
- /// Call add() until you have added all the mappings, then call build() to get a NormalizeCharMap
- /// @lucene.experimental
- /// </para>
- /// </summary>
- public class Builder
- {
+ /// <summary>
+ /// Builds an NormalizeCharMap.
+ /// <para>
+ /// Call add() until you have added all the mappings, then call build() to get a NormalizeCharMap
+ /// @lucene.experimental
+ /// </para>
+ /// </summary>
+ public class Builder
+ {
- internal readonly IDictionary<string, string> pendingPairs = new SortedDictionary<string, string>();
+ internal readonly IDictionary<string, string> pendingPairs = new SortedDictionary<string, string>();
- /// <summary>
- /// Records a replacement to be applied to the input
- /// stream. Whenever <code>singleMatch</code> occurs in
- /// the input, it will be replaced with
- /// <code>replacement</code>.
- /// </summary>
- /// <param name="match"> input String to be replaced </param>
- /// <param name="replacement"> output String </param>
- /// <exception cref="IllegalArgumentException"> if
- /// <code>match</code> is the empty string, or was
- /// already previously added </exception>
- public virtual void add(string match, string replacement)
- {
- if (match.Length == 0)
- {
- throw new System.ArgumentException("cannot match the empty string");
- }
- if (pendingPairs.ContainsKey(match))
- {
- throw new System.ArgumentException("match \"" + match + "\" was already added");
- }
- pendingPairs[match] = replacement;
- }
+ /// <summary>
+ /// Records a replacement to be applied to the input
+ /// stream. Whenever <code>singleMatch</code> occurs in
+ /// the input, it will be replaced with
+ /// <code>replacement</code>.
+ /// </summary>
+ /// <param name="match"> input String to be replaced </param>
+ /// <param name="replacement"> output String </param>
+ /// <exception cref="IllegalArgumentException"> if
+ /// <code>match</code> is the empty string, or was
+ /// already previously added </exception>
+ public virtual void Add(string match, string replacement)
+ {
+ if (match.Length == 0)
+ {
+ throw new System.ArgumentException("cannot match the empty string");
+ }
+ if (pendingPairs.ContainsKey(match))
+ {
+ throw new System.ArgumentException("match \"" + match + "\" was already added");
+ }
+ pendingPairs[match] = replacement;
+ }
- /// <summary>
- /// Builds the NormalizeCharMap; call this once you
- /// are done calling <seealso cref="#add"/>.
- /// </summary>
- public virtual NormalizeCharMap build()
- {
+ /// <summary>
+ /// Builds the NormalizeCharMap; call this once you
+ /// are done calling <seealso cref="#add"/>.
+ /// </summary>
+ public virtual NormalizeCharMap build()
+ {
- FST<CharsRef> map;
- try
- {
- Outputs<CharsRef> outputs = CharSequenceOutputs.Singleton;
- Builder<CharsRef> builder = new Builder<CharsRef>(FST.INPUT_TYPE.BYTE2, outputs);
- IntsRef scratch = new IntsRef();
- foreach (KeyValuePair<string, string> ent in pendingPairs.SetOfKeyValuePairs())
- {
- builder.Add(Util.ToUTF16(ent.Key, scratch), new CharsRef(ent.Value));
- }
- map = builder.Finish();
- pendingPairs.Clear();
- }
- catch (IOException ioe)
- {
- // Bogus FST IOExceptions!! (will never happen)
- throw new Exception(ioe);
- }
+ FST<CharsRef> map;
+ try
+ {
+ Outputs<CharsRef> outputs = CharSequenceOutputs.Singleton;
+ Builder<CharsRef> builder = new Builder<CharsRef>(FST.INPUT_TYPE.BYTE2, outputs);
+ IntsRef scratch = new IntsRef();
+ foreach (var ent in pendingPairs)
+ {
+ builder.Add(Util.ToUTF16(ent.Key, scratch), new CharsRef(ent.Value));
+ }
+ map = builder.Finish();
+ pendingPairs.Clear();
+ }
+ catch (IOException ioe)
+ {
+ // Bogus FST IOExceptions!! (will never happen)
+ throw new Exception("Should never happen", ioe);
+ }
- return new NormalizeCharMap(map);
- }
- }
- }
+ return new NormalizeCharMap(map);
+ }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Core/LetterTokenizer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Core/LetterTokenizer.cs b/src/Lucene.Net.Analysis.Common/Analysis/Core/LetterTokenizer.cs
index de59e18..3ce712c 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Core/LetterTokenizer.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Core/LetterTokenizer.cs
@@ -76,7 +76,7 @@ namespace Lucene.Net.Analysis.Core
/// Collects only characters which satisfy
/// <seealso cref="Character#isLetter(int)"/>.
/// </summary>
- protected override bool IsTokenChar(int c)
+ protected override bool IsTokenChar(char c)
{
return Character.IsLetter(c);
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/HyphenatedWordsFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/HyphenatedWordsFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/HyphenatedWordsFilter.cs
index d423316..d4d9c65 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/HyphenatedWordsFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/HyphenatedWordsFilter.cs
@@ -1,4 +1,5 @@
-using System.Text;
+using System;
+using System.Text;
using Lucene.Net.Analysis.Tokenattributes;
namespace Lucene.Net.Analysis.Miscellaneous
@@ -149,7 +150,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
term = termAttribute.ResizeBuffer(length);
}
- hyphenated.GetChars(0, length, term, 0);
+ term = hyphenated.ToString().ToCharArray();
termAttribute.Length = length;
offsetAttribute.SetOffset(offsetAttribute.StartOffset(), lastEndOffset);
hyphenated.Length = 0;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/LimitTokenCountAnalyzer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/LimitTokenCountAnalyzer.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/LimitTokenCountAnalyzer.cs
index 585accd..f8cb2da 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/LimitTokenCountAnalyzer.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/LimitTokenCountAnalyzer.cs
@@ -46,7 +46,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
/// <param name="maxTokenCount"> max number of tokens to produce </param>
/// <param name="consumeAllTokens"> whether all tokens from the delegate should be consumed even if maxTokenCount is reached. </param>
public LimitTokenCountAnalyzer(Analyzer @delegate, int maxTokenCount, bool consumeAllTokens)
- : base(@delegate.ReuseStrategy)
+ : base(@delegate.Strategy)
{
this.@delegate = @delegate;
this.maxTokenCount = maxTokenCount;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PerFieldAnalyzerWrapper.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PerFieldAnalyzerWrapper.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PerFieldAnalyzerWrapper.cs
index 52c8ded..d43a407 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PerFieldAnalyzerWrapper.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PerFieldAnalyzerWrapper.cs
@@ -77,7 +77,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
: base(PER_FIELD_REUSE_STRATEGY)
{
this.defaultAnalyzer = defaultAnalyzer;
- this.fieldAnalyzers = (fieldAnalyzers != null) ? fieldAnalyzers : System.Linq.Enumerable.Empty<string, Analyzer>();
+ this.fieldAnalyzers = fieldAnalyzers ?? new Dictionary<string, Analyzer>();
}
protected override Analyzer GetWrappedAnalyzer(string fieldName)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PrefixAndSuffixAwareTokenFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PrefixAndSuffixAwareTokenFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PrefixAndSuffixAwareTokenFilter.cs
index df7bf38..c57e0c1 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PrefixAndSuffixAwareTokenFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/PrefixAndSuffixAwareTokenFilter.cs
@@ -4,7 +4,7 @@ using Lucene.Net.Analysis.Miscellaneous;
namespace org.apache.lucene.analysis.miscellaneous
{
- /*
+ /*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -22,89 +22,84 @@ namespace org.apache.lucene.analysis.miscellaneous
*/
- /// <summary>
- /// Links two <seealso cref="PrefixAwareTokenFilter"/>.
- /// <p/>
- /// <b>NOTE:</b> This filter might not behave correctly if used with custom Attributes, i.e. Attributes other than
- /// the ones located in org.apache.lucene.analysis.tokenattributes.
- /// </summary>
- public class PrefixAndSuffixAwareTokenFilter : TokenStream
- {
-
- private readonly PrefixAwareTokenFilter suffix;
-
- public PrefixAndSuffixAwareTokenFilter(TokenStream prefix, TokenStream input, TokenStream suffix) : base(suffix)
- {
- prefix = new PrefixAwareTokenFilterAnonymousInnerClassHelper(this, prefix, input);
- this.suffix = new PrefixAwareTokenFilterAnonymousInnerClassHelper2(this, prefix, suffix);
- }
-
- private class PrefixAwareTokenFilterAnonymousInnerClassHelper : PrefixAwareTokenFilter
- {
- private readonly PrefixAndSuffixAwareTokenFilter outerInstance;
-
- public PrefixAwareTokenFilterAnonymousInnerClassHelper(PrefixAndSuffixAwareTokenFilter outerInstance, TokenStream prefix, TokenStream input) : base(prefix, input)
- {
- this.outerInstance = outerInstance;
- }
-
- public override Token UpdateSuffixToken(Token suffixToken, Token lastInputToken)
- {
- return outerInstance.updateInputToken(suffixToken, lastInputToken);
- }
- }
-
- private class PrefixAwareTokenFilterAnonymousInnerClassHelper2 : PrefixAwareTokenFilter
- {
- private readonly PrefixAndSuffixAwareTokenFilter outerInstance;
-
- public PrefixAwareTokenFilterAnonymousInnerClassHelper2(PrefixAndSuffixAwareTokenFilter outerInstance, TokenStream prefix, TokenStream suffix) : base(prefix, suffix)
- {
- this.outerInstance = outerInstance;
- }
-
- public override Token UpdateSuffixToken(Token suffixToken, Token lastInputToken)
- {
- return outerInstance.updateSuffixToken(suffixToken, lastInputToken);
- }
- }
-
- public virtual Token updateInputToken(Token inputToken, Token lastPrefixToken)
- {
- inputToken.SetOffset(lastPrefixToken.endOffset() + inputToken.startOffset(), lastPrefixToken.endOffset() + inputToken.endOffset());
- return inputToken;
- }
-
- public virtual Token updateSuffixToken(Token suffixToken, Token lastInputToken)
- {
- suffixToken.SetOffset(lastInputToken.endOffset() + suffixToken.startOffset(), lastInputToken.endOffset() + suffixToken.endOffset());
- return suffixToken;
- }
-
- public override bool IncrementToken()
- {
- return suffix.incrementToken();
- }
-
- public override void Reset()
- {
- suffix.Reset();
- }
-
-
-//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
-//ORIGINAL LINE: @Override public void close() throws java.io.IOException
- public override void close()
- {
- suffix.close();
- }
-
-//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
-//ORIGINAL LINE: @Override public void end() throws java.io.IOException
- public override void end()
- {
- suffix.end();
- }
- }
-
+ /// <summary>
+ /// Links two <seealso cref="PrefixAwareTokenFilter"/>.
+ /// <p/>
+ /// <b>NOTE:</b> This filter might not behave correctly if used with custom Attributes, i.e. Attributes other than
+ /// the ones located in org.apache.lucene.analysis.tokenattributes.
+ /// </summary>
+ public class PrefixAndSuffixAwareTokenFilter : TokenStream
+ {
+
+ private readonly PrefixAwareTokenFilter suffix;
+
+ public PrefixAndSuffixAwareTokenFilter(TokenStream prefix, TokenStream input, TokenStream suffix) : base(suffix)
+ {
+ prefix = new PrefixAwareTokenFilterAnonymousInnerClassHelper(this, prefix, input);
+ this.suffix = new PrefixAwareTokenFilterAnonymousInnerClassHelper2(this, prefix, suffix);
+ }
+
+ private class PrefixAwareTokenFilterAnonymousInnerClassHelper : PrefixAwareTokenFilter
+ {
+ private readonly PrefixAndSuffixAwareTokenFilter outerInstance;
+
+ public PrefixAwareTokenFilterAnonymousInnerClassHelper(PrefixAndSuffixAwareTokenFilter outerInstance, TokenStream prefix, TokenStream input) : base(prefix, input)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ public override Token UpdateSuffixToken(Token suffixToken, Token lastInputToken)
+ {
+ return outerInstance.updateInputToken(suffixToken, lastInputToken);
+ }
+ }
+
+ private class PrefixAwareTokenFilterAnonymousInnerClassHelper2 : PrefixAwareTokenFilter
+ {
+ private readonly PrefixAndSuffixAwareTokenFilter outerInstance;
+
+ public PrefixAwareTokenFilterAnonymousInnerClassHelper2(PrefixAndSuffixAwareTokenFilter outerInstance, TokenStream prefix, TokenStream suffix) : base(prefix, suffix)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ public override Token UpdateSuffixToken(Token suffixToken, Token lastInputToken)
+ {
+ return outerInstance.updateSuffixToken(suffixToken, lastInputToken);
+ }
+ }
+
+ public virtual Token updateInputToken(Token inputToken, Token lastPrefixToken)
+ {
+ inputToken.SetOffset(lastPrefixToken.EndOffset() + inputToken.StartOffset(), lastPrefixToken.EndOffset() + inputToken.EndOffset());
+ return inputToken;
+ }
+
+ public virtual Token updateSuffixToken(Token suffixToken, Token lastInputToken)
+ {
+ suffixToken.SetOffset(lastInputToken.EndOffset() + suffixToken.StartOffset(), lastInputToken.EndOffset() + suffixToken.EndOffset());
+ return suffixToken;
+ }
+
+ public override bool IncrementToken()
+ {
+ return suffix.IncrementToken();
+ }
+
+ public override void Reset()
+ {
+ suffix.Reset();
+ }
+
+
+ public override void Dispose()
+ {
+ suffix.Dispose();
+ }
+
+ public override void End()
+ {
+ suffix.End();
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/SetKeywordMarkerFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/SetKeywordMarkerFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/SetKeywordMarkerFilter.cs
index c21607b..7639668 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/SetKeywordMarkerFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/SetKeywordMarkerFilter.cs
@@ -1,6 +1,4 @@
using Lucene.Net.Analysis.Tokenattributes;
-using Lucene.Net.Analysis.Util;
-using org.apache.lucene.analysis.miscellaneous;
namespace Lucene.Net.Analysis.Miscellaneous
{
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilter.cs
index 9382516..95bd4d1 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilter.cs
@@ -1,4 +1,8 @@
using System;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Util;
+using Version = Lucene.Net.Util.LuceneVersion;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -19,10 +23,6 @@
namespace org.apache.lucene.analysis.reverse
{
-
- using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
- using Version = org.apache.lucene.util.Version;
-
/// <summary>
/// Reverse token string, for example "country" => "yrtnuoc".
/// <para>
@@ -43,7 +43,7 @@ namespace org.apache.lucene.analysis.reverse
public sealed class ReverseStringFilter : TokenFilter
{
- private readonly CharTermAttribute termAtt = addAttribute(typeof(CharTermAttribute));
+ private readonly CharTermAttribute termAtt;
private readonly char marker;
private readonly Version matchVersion;
private const char NOMARKER = '\uFFFF';
@@ -98,20 +98,18 @@ namespace org.apache.lucene.analysis.reverse
this.marker = marker;
}
-//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
-//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
- public override bool incrementToken()
+ public override bool IncrementToken()
{
- if (input.incrementToken())
+ if (input.IncrementToken())
{
- int len = termAtt.length();
+ int len = termAtt.Length;
if (marker != NOMARKER)
{
len++;
- termAtt.resizeBuffer(len);
- termAtt.buffer()[len - 1] = marker;
+ termAtt.ResizeBuffer(len);
+ termAtt.Buffer()[len - 1] = marker;
}
- reverse(matchVersion, termAtt.buffer(), 0, len);
+ reverse(matchVersion, termAtt.Buffer(), 0, len);
termAtt.Length = len;
return true;
}
@@ -188,11 +186,9 @@ namespace org.apache.lucene.analysis.reverse
/// <param name="start"> the offset from where to reverse the buffer </param>
/// <param name="len"> the length in the buffer up to where the
/// buffer should be reversed </param>
-//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
-//ORIGINAL LINE: public static void reverse(org.apache.lucene.util.Version matchVersion, final char[] buffer, final int start, final int len)
public static void reverse(Version matchVersion, char[] buffer, int start, int len)
{
- if (!matchVersion.onOrAfter(Version.LUCENE_31))
+ if (!matchVersion.OnOrAfter(Version.LUCENE_31))
{
reverseUnicode3(buffer, start, len);
return;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilterFactory.cs b/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilterFactory.cs
index abc8d24..a78c23a 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilterFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Reverse/ReverseStringFilterFactory.cs
@@ -1,7 +1,7 @@
using System.Collections.Generic;
-using TokenFilterFactory = Lucene.Net.Analysis.Util.TokenFilterFactory;
+using org.apache.lucene.analysis.reverse;
-namespace org.apache.lucene.analysis.reverse
+namespace Lucene.Net.Analysis.Reverse
{
/*
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Util/AbstractAnalysisFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Util/AbstractAnalysisFactory.cs b/src/Lucene.Net.Analysis.Common/Analysis/Util/AbstractAnalysisFactory.cs
index 27d2531..b62a110 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Util/AbstractAnalysisFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Util/AbstractAnalysisFactory.cs
@@ -4,7 +4,6 @@ using System.IO;
using Lucene.Net.Analysis.Core;
using Lucene.Net.Support;
using Lucene.Net.Util;
-using org.apache.lucene.analysis.util;
using Reader = System.IO.TextReader;
using Version = Lucene.Net.Util.LuceneVersion;
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Analysis/Util/CharTokenizer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Util/CharTokenizer.cs b/src/Lucene.Net.Analysis.Common/Analysis/Util/CharTokenizer.cs
index 19772ee..37b1921 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Util/CharTokenizer.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Util/CharTokenizer.cs
@@ -114,7 +114,7 @@ namespace Lucene.Net.Analysis.Util
/// predicate. Codepoints for which this is false are used to define token
/// boundaries and are not included in tokens.
/// </summary>
- protected abstract bool IsTokenChar(int c);
+ protected abstract bool IsTokenChar(char c);
/// <summary>
/// Called on each token character to normalize it before it is added to the
@@ -160,7 +160,7 @@ namespace Lucene.Net.Analysis.Util
int charCount = Character.CharCount(c);
bufferIndex += charCount;
- if (IsTokenChar(c)) // if it's a token char
+ if (IsTokenChar((char)c)) // if it's a token char
{
if (length == 0) // start of token
{
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Analysis.Common/Lucene.Net.Analysis.Common.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Lucene.Net.Analysis.Common.csproj b/src/Lucene.Net.Analysis.Common/Lucene.Net.Analysis.Common.csproj
index e0c7b3b..d8516e2 100644
--- a/src/Lucene.Net.Analysis.Common/Lucene.Net.Analysis.Common.csproj
+++ b/src/Lucene.Net.Analysis.Common/Lucene.Net.Analysis.Common.csproj
@@ -38,13 +38,35 @@
<HintPath>..\..\packages\ICU4NET-ICU4C55.1-bin32.1.0.0\lib\net45\ICU4NETExtension.dll</HintPath>
<Private>True</Private>
</Reference>
+ <Reference Include="Microsoft.VisualBasic" />
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="Microsoft.CSharp" />
</ItemGroup>
<ItemGroup>
+ <Compile Include="Analysis\CharFilter\BaseCharFilter.cs" />
+ <Compile Include="Analysis\Core\KeywordAnalyzer.cs" />
+ <Compile Include="Analysis\Core\KeywordTokenizer.cs" />
<Compile Include="Analysis\Core\LetterTokenizer.cs" />
+ <Compile Include="Analysis\Core\LowerCaseFilter.cs" />
<Compile Include="Analysis\Core\LowerCaseTokenizer.cs" />
+ <Compile Include="Analysis\Core\SimpleAnalyzer.cs" />
+ <Compile Include="Analysis\Miscellaneous\ASCIIFoldingFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\EmptyTokenStream.cs" />
+ <Compile Include="Analysis\Miscellaneous\HyphenatedWordsFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\KeywordMarkerFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\KeywordRepeatFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\LimitTokenCountAnalyzer.cs" />
+ <Compile Include="Analysis\Miscellaneous\LimitTokenCountFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\LimitTokenPositionFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\PerFieldAnalyzerWrapper.cs" />
+ <Compile Include="Analysis\Miscellaneous\PrefixAndSuffixAwareTokenFilter.cs" />
+ <Compile Include="Analysis\Miscellaneous\PrefixAwareTokenFilter.cs" />
+ <Compile Include="Analysis\Reverse\ReverseStringFilter.cs" />
+ <Compile Include="Analysis\Util\StemmerUtil.cs" />
+ <Compile Include="Analysis\Core\UpperCaseFilter.cs" />
+ <Compile Include="Analysis\Core\WhitespaceAnalyzer.cs" />
+ <Compile Include="Analysis\Core\WhitespaceTokenizer.cs" />
<Compile Include="Analysis\Util\CharacterIterator.cs" />
<Compile Include="Analysis\Util\CharacterUtils.cs">
<SubType>Code</SubType>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8e144a0c/src/Lucene.Net.Core/Analysis/CharFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/CharFilter.cs b/src/Lucene.Net.Core/Analysis/CharFilter.cs
index bae10b1..4cc8536 100644
--- a/src/Lucene.Net.Core/Analysis/CharFilter.cs
+++ b/src/Lucene.Net.Core/Analysis/CharFilter.cs
@@ -69,7 +69,7 @@ namespace Lucene.Net.Analysis
/// </summary>
/// <param name="currentOff"> current offset </param>
/// <returns> corrected offset </returns>
- protected internal abstract int Correct(int currentOff);
+ protected abstract int Correct(int currentOff);
/// <summary>
/// Chains the corrected offset through the input