You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by pa...@apache.org on 2026/04/11 03:14:56 UTC
(lucenenet) branch master updated: BREAKING: More Span/stackalloc optimizations (#1156)
This is an automated email from the ASF dual-hosted git repository.
paulirwin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
The following commit(s) were added to refs/heads/master by this push:
new f623208d4 BREAKING: More Span/stackalloc optimizations (#1156)
f623208d4 is described below
commit f623208d46946c3474568b6f00539ec209130171
Author: Paul Irwin <pa...@gmail.com>
AuthorDate: Fri Apr 10 21:14:52 2026 -0600
BREAKING: More Span/stackalloc optimizations (#1156)
* Use Span in ChainedFilter
* Use Span in TernaryTree
* Use Span in HyphenationTree
* Remove allocation in PatternParser
* Use Span in PayloadHelper Decode methods
* Remove allocation in GetCCByGB2312Id
* Use Span in Egothor.Stemmer.Diff
* PR review changes
---
Directory.Build.targets | 1 +
.../Compound/Hyphenation/HyphenationTree.cs | 27 ++++++------
.../Analysis/Compound/Hyphenation/PatternParser.cs | 7 ++--
.../Analysis/Compound/Hyphenation/TernaryTree.cs | 48 ++++++++++++++--------
.../Analysis/Payloads/PayloadHelper.cs | 12 +++---
.../Hhmm/AbstractDictionary.cs | 6 ++-
.../Egothor.Stemmer/Diff.cs | 2 +-
src/Lucene.Net.Queries/ChainedFilter.cs | 9 ++--
.../Support/Text/TestEncodingExtensions.cs | 15 +++++++
src/Lucene.Net/Support/Text/EncodingExtensions.cs | 15 +++++++
10 files changed, 100 insertions(+), 42 deletions(-)
diff --git a/Directory.Build.targets b/Directory.Build.targets
index e090c18b8..48ad23bc2 100644
--- a/Directory.Build.targets
+++ b/Directory.Build.targets
@@ -94,6 +94,7 @@
<DefineConstants>$(DefineConstants);FEATURE_ARRAY_FILL</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_CONDITIONALWEAKTABLE_ADDORUPDATE</DefineConstants>
+ <DefineConstants>$(DefineConstants);FEATURE_ENCODING_GETSTRING_READONLYSPAN</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_MEMORYMARSHAL_CREATEREADONLYSPAN</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_NUMBER_PARSE_READONLYSPAN</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_STREAM_READ_SPAN</DefineConstants>
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/HyphenationTree.cs b/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/HyphenationTree.cs
index d94f0afcd..a13f45588 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/HyphenationTree.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/HyphenationTree.cs
@@ -1,5 +1,7 @@
// Lucene version compatibility level 4.8.1
using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
@@ -220,7 +222,7 @@ public virtual string FindPattern(string pat)
/// <summary>
/// String compare, returns 0 if equal or t is a substring of s
/// </summary>
- protected virtual int HStrCmp(char[] s, int si, char[] t, int ti)
+ protected virtual int HStrCmp(ReadOnlySpan<char> s, int si, ReadOnlySpan<char> t, int ti)
{
for (; s[si] == t[ti]; si++, ti++)
{
@@ -287,7 +289,7 @@ protected virtual byte[] GetValues(int k)
/// <param name="word"> null terminated word to match </param>
/// <param name="index"> start index from word </param>
/// <param name="il"> interletter values array to update </param>
- protected virtual void SearchPatterns(char[] word, int index, byte[] il)
+ protected virtual void SearchPatterns(ReadOnlySpan<char> word, int index, Span<byte> il)
{
byte[] values;
int i = index;
@@ -375,8 +377,8 @@ protected virtual void SearchPatterns(char[] word, int index, byte[] il)
/// hyphenated word or null if word is not hyphenated. </returns>
public virtual Hyphenation Hyphenate(string word, int remainCharCount, int pushCharCount)
{
- char[] w = word.ToCharArray();
- return Hyphenate(w, 0, w.Length, remainCharCount, pushCharCount);
+ // LUCENENET: use Span instead of ToCharArray
+ return Hyphenate(word.AsSpan(), 0, word.Length, remainCharCount, pushCharCount);
}
@@ -406,13 +408,14 @@ public virtual Hyphenation Hyphenate(string word, int remainCharCount, int pushC
/// hyphenation point. </param>
/// <returns> a <see cref="Hyphenation"/> object representing the
/// hyphenated word or null if word is not hyphenated. </returns>
- public virtual Hyphenation Hyphenate(char[] w, int offset, int len, int remainCharCount, int pushCharCount)
+ public virtual Hyphenation Hyphenate(ReadOnlySpan<char> w, int offset, int len, int remainCharCount, int pushCharCount)
{
int i;
- char[] word = new char[len + 3];
+ // LUCENENET: optimized method for Span and stackalloc
+ Span<char> word = (len + 3) * sizeof(char) > Constants.MaxStackByteLimit ? new char[len + 3] : stackalloc char[len + 3];
// normalize word
- char[] c = new char[2];
+ Span<char> c = stackalloc char[2];
int iIgnoreAtBeginning = 0;
int iLength = len;
bool bEndOfLetters = false;
@@ -452,11 +455,11 @@ public virtual Hyphenation Hyphenate(char[] w, int offset, int len, int remainCh
// word is too short to be hyphenated
return null;
}
- int[] result = new int[len + 1];
+ Span<int> result = (len + 1) * sizeof(int) > Constants.MaxStackByteLimit ? new int[len + 1] : stackalloc int[len + 1];
int k = 0;
// check exception list first
- string sw = new string(word, 1, len);
+ string sw = word.Slice(1, len).ToString();
// LUCENENET: Eliminated extra lookup by using TryGetValue instead of ContainsKey
if (m_stoplist.TryGetValue(sw, out IList<object> hw))
{
@@ -484,7 +487,7 @@ public virtual Hyphenation Hyphenate(char[] w, int offset, int len, int remainCh
word[0] = '.'; // word start marker
word[len + 1] = '.'; // word end marker
word[len + 2] = (char)0; // null terminated
- byte[] il = new byte[len + 3]; // initialized to zero
+ Span<byte> il = (len + 3) > Constants.MaxStackByteLimit ? new byte[len + 3] : stackalloc byte[len + 3]; // initialized to zero
for (i = 0; i < len + 1; i++)
{
SearchPatterns(word, i, il);
@@ -507,7 +510,7 @@ public virtual Hyphenation Hyphenate(char[] w, int offset, int len, int remainCh
{
// trim result array
int[] res = new int[k + 2];
- Arrays.Copy(result, 0, res, 1, k);
+ result.Slice(0, k).CopyTo(res.AsSpan(1, k));
// We add the synthetical hyphenation points
// at the beginning and end of the word
res[0] = 0;
@@ -535,7 +538,7 @@ public virtual void AddClass(string chargroup)
if (chargroup.Length > 0)
{
char equivChar = chargroup[0];
- char[] key = new char[2];
+ Span<char> key = stackalloc char[2];
key[1] = (char)0;
for (int i = 0; i < chargroup.Length; i++)
{
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/PatternParser.cs b/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/PatternParser.cs
index 2d3b77941..f55a849e7 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/PatternParser.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/PatternParser.cs
@@ -299,11 +299,12 @@ protected virtual IList<object> NormalizeException<T1>(IList<T1> ex)
{
res.Add(buf.ToString());
buf.Length = 0;
- char[] h = new char[1];
- h[0] = hyphenChar;
+ // LUCENENET: commented out unnecessary array allocation
+ //char[] h = new char[1];
+ //h[0] = hyphenChar;
// we use here hyphenChar which is not necessarily
// the one to be printed
- res.Add(new Hyphen(new string(h), null, null));
+ res.Add(new Hyphen(new string(hyphenChar, 1), null, null));
}
}
if (buf.Length > 0)
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/TernaryTree.cs b/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/TernaryTree.cs
index 29512e124..b19ed66b7 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/TernaryTree.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Compound/Hyphenation/TernaryTree.cs
@@ -1,5 +1,6 @@
// Lucene version compatibility level 4.8.1
using Lucene.Net.Support;
+using Lucene.Net.Util;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -145,7 +146,19 @@ protected void Init()
/// same prefix is inserted. This saves a lot of space, specially for long
/// keys.
/// </summary>
- public virtual void Insert(string key, char val)
+ public void Insert(string key, char val)
+ => Insert(key.AsSpan(), val);
+
+ /// <summary>
+ /// Branches are initially compressed, needing one node per key plus the size
+ /// of the string key. They are decompressed as needed when another key with
+ /// same prefix is inserted. This saves a lot of space, specially for long
+ /// keys.
+ /// </summary>
+ /// <remarks>
+ /// LUCENENET specific overload for <see cref="ReadOnlySpan{T}"/>
+ /// </remarks>
+ public virtual void Insert(ReadOnlySpan<char> key, char val)
{
// make sure we have enough room in the arrays
int len = key.Length + 1; // maximum number of nodes that may be generated
@@ -153,13 +166,15 @@ public virtual void Insert(string key, char val)
{
RedimNodeArrays(m_eq.Length + BLOCK_SIZE);
}
- char[] strkey = new char[len--];
- key.CopyTo(0, strkey, 0, len - 0);
+ // LUCENENET: add optimization for stackalloc and Span
+ int strkeyLen = len--;
+ Span<char> strkey = strkeyLen * sizeof(char) > Constants.MaxStackByteLimit ? new char[strkeyLen] : stackalloc char[strkeyLen];
+ key.CopyTo(strkey);
strkey[len] = (char)0;
m_root = Insert(m_root, strkey, 0, val);
}
- public virtual void Insert(char[] key, int start, char val)
+ public virtual void Insert(ReadOnlySpan<char> key, int start, char val)
{
int len = StrLen(key) + 1;
if (m_freenode + len > m_eq.Length)
@@ -172,7 +187,7 @@ public virtual void Insert(char[] key, int start, char val)
/// <summary>
/// The actual insertion function, recursive version.
/// </summary>
- private char Insert(char p, char[] key, int start, char val)
+ private char Insert(char p, ReadOnlySpan<char> key, int start, char val)
{
int len = StrLen(key, start);
if (p == 0)
@@ -264,7 +279,7 @@ private char Insert(char p, char[] key, int start, char val)
/// <summary>
/// Compares 2 null terminated char arrays
/// </summary>
- public static int StrCmp(char[] a, int startA, char[] b, int startB)
+ public static int StrCmp(ReadOnlySpan<char> a, int startA, ReadOnlySpan<char> b, int startB)
{
for (; a[startA] == b[startB]; startA++, startB++)
{
@@ -279,7 +294,7 @@ public static int StrCmp(char[] a, int startA, char[] b, int startB)
/// <summary>
/// Compares a string with null terminated char array
/// </summary>
- public static int StrCmp(string str, char[] a, int start)
+ public static int StrCmp(ReadOnlySpan<char> str, ReadOnlySpan<char> a, int start)
{
int i, d, len = str.Length;
for (i = 0; i < len; i++)
@@ -299,10 +314,9 @@ public static int StrCmp(string str, char[] a, int start)
return -a[start + i];
}
return 0;
-
}
- public static void StrCpy(char[] dst, int di, char[] src, int si)
+ public static void StrCpy(Span<char> dst, int di, ReadOnlySpan<char> src, int si)
{
while (src[si] != 0)
{
@@ -311,7 +325,7 @@ public static void StrCpy(char[] dst, int di, char[] src, int si)
dst[di] = (char)0;
}
- public static int StrLen(char[] a, int start)
+ public static int StrLen(ReadOnlySpan<char> a, int start)
{
int len = 0;
for (int i = start; i < a.Length && a[i] != 0; i++)
@@ -321,7 +335,7 @@ public static int StrLen(char[] a, int start)
return len;
}
- public static int StrLen(char[] a)
+ public static int StrLen(ReadOnlySpan<char> a)
{
return StrLen(a, 0);
}
@@ -329,14 +343,15 @@ public static int StrLen(char[] a)
public virtual int Find(string key)
{
int len = key.Length;
- char[] strkey = new char[len + 1];
- key.CopyTo(0, strkey, 0, len - 0);
+ // LUCENENET: add optimization for stackalloc and Span
+ Span<char> strkey = (len + 1) * sizeof(char) > Constants.MaxStackByteLimit ? new char[len + 1] : stackalloc char[len + 1];
+ key.AsSpan().CopyTo(strkey);
strkey[len] = (char)0;
return Find(strkey, 0);
}
- public virtual int Find(char[] key, int start)
+ public virtual int Find(ReadOnlySpan<char> key, int start)
{
int d;
char p = m_root;
@@ -424,7 +439,7 @@ public virtual object Clone()
/// upper halves, and so on in order to get a balanced tree. The array of keys
/// is assumed to be sorted in ascending order.
/// </summary>
- protected virtual void InsertBalanced(string[] k, char[] v, int offset, int n)
+ protected virtual void InsertBalanced(ReadOnlySpan<string> k, ReadOnlySpan<char> v, int offset, int n)
{
int m;
if (n < 1)
@@ -449,7 +464,8 @@ public virtual void Balance()
int i = 0, n = m_length;
string[] k = new string[n];
- char[] v = new char[n];
+ // LUCENENET: add optimization for stackalloc and Span
+ Span<char> v = n * sizeof(char) > Constants.MaxStackByteLimit ? new char[n] : stackalloc char[n];
using (Enumerator iter = new Enumerator(this))
{
while (iter.MoveNext())
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Payloads/PayloadHelper.cs b/src/Lucene.Net.Analysis.Common/Analysis/Payloads/PayloadHelper.cs
index fb2d3e074..9e4cc875f 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Payloads/PayloadHelper.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Payloads/PayloadHelper.cs
@@ -1,4 +1,7 @@
// Lucene version compatibility level 4.8.1
+
+using System;
+
namespace Lucene.Net.Analysis.Payloads
{
/*
@@ -62,10 +65,10 @@ public static byte[] EncodeInt32(int payload, byte[] data, int offset)
/// <summary>
/// NOTE: This was decodeFloat() in Lucene
/// </summary>
- /// <seealso cref="DecodeSingle(byte[], int)"/>
+ /// <seealso cref="DecodeSingle(ReadOnlySpan{byte}, int)"/>
/// <seealso cref="EncodeSingle(float)"/>
/// <returns> the decoded float </returns>
- public static float DecodeSingle(byte[] bytes)
+ public static float DecodeSingle(ReadOnlySpan<byte> bytes)
{
return DecodeSingle(bytes, 0);
}
@@ -81,16 +84,15 @@ public static float DecodeSingle(byte[] bytes)
/// <returns> The float that was encoded
/// </returns>
/// <seealso cref="EncodeSingle(float)"/>
- public static float DecodeSingle(byte[] bytes, int offset)
+ public static float DecodeSingle(ReadOnlySpan<byte> bytes, int offset)
{
-
return J2N.BitConversion.Int32BitsToSingle(DecodeInt32(bytes, offset));
}
/// <summary>
/// NOTE: This was decodeInt() in Lucene
/// </summary>
- public static int DecodeInt32(byte[] bytes, int offset)
+ public static int DecodeInt32(ReadOnlySpan<byte> bytes, int offset)
{
return ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
}
diff --git a/src/Lucene.Net.Analysis.SmartCn/Hhmm/AbstractDictionary.cs b/src/Lucene.Net.Analysis.SmartCn/Hhmm/AbstractDictionary.cs
index cdfb7154d..95de24752 100644
--- a/src/Lucene.Net.Analysis.SmartCn/Hhmm/AbstractDictionary.cs
+++ b/src/Lucene.Net.Analysis.SmartCn/Hhmm/AbstractDictionary.cs
@@ -2,6 +2,10 @@
using System;
using System.Text;
+#if !FEATURE_ENCODING_GETSTRING_READONLYSPAN
+using Lucene.Net.Support.Text;
+#endif
+
namespace Lucene.Net.Analysis.Cn.Smart.Hhmm
{
/*
@@ -97,7 +101,7 @@ public virtual string GetCCByGB2312Id(int ccid)
return "";
int cc1 = ccid / 94 + 161;
int cc2 = ccid % 94 + 161;
- byte[] buffer = new byte[2];
+ Span<byte> buffer = stackalloc byte[2];
buffer[0] = (byte)cc1;
buffer[1] = (byte)cc2;
try
diff --git a/src/Lucene.Net.Analysis.Stempel/Egothor.Stemmer/Diff.cs b/src/Lucene.Net.Analysis.Stempel/Egothor.Stemmer/Diff.cs
index b93468270..08070937c 100644
--- a/src/Lucene.Net.Analysis.Stempel/Egothor.Stemmer/Diff.cs
+++ b/src/Lucene.Net.Analysis.Stempel/Egothor.Stemmer/Diff.cs
@@ -185,7 +185,7 @@ public string Exec(string a, string b)
int y;
int maxx;
int maxy;
- int[] go = new int[4];
+ Span<int> go = stackalloc int[4]; // LUCENENET: optimize for Span and stackalloc
const int X = 1;
const int Y = 2;
const int R = 3;
diff --git a/src/Lucene.Net.Queries/ChainedFilter.cs b/src/Lucene.Net.Queries/ChainedFilter.cs
index aaa3540c6..399c06132 100644
--- a/src/Lucene.Net.Queries/ChainedFilter.cs
+++ b/src/Lucene.Net.Queries/ChainedFilter.cs
@@ -96,7 +96,8 @@ public ChainedFilter(Filter[] chain, int logic)
/// </summary>
public override DocIdSet GetDocIdSet(AtomicReaderContext context, IBits acceptDocs)
{
- int[] index = new int[1]; // use array as reference to modifiable int;
+ // LUCENENET specific - use stackalloc and Span instead of new int[1]
+ Span<int> index = stackalloc int[1]; // use array as reference to modifiable int;
index[0] = 0; // an object attribute would not be thread safe.
if (logic != -1)
{
@@ -132,7 +133,7 @@ public override DocIdSet GetDocIdSet(AtomicReaderContext context, IBits acceptDo
}
}
- private FixedBitSet InitialResult(AtomicReaderContext context, int logic, int[] index)
+ private FixedBitSet InitialResult(AtomicReaderContext context, int logic, Span<int> index)
{
AtomicReader reader = context.AtomicReader;
FixedBitSet result = new FixedBitSet(reader.MaxDoc);
@@ -157,7 +158,7 @@ private FixedBitSet InitialResult(AtomicReaderContext context, int logic, int[]
/// <param name="logic"> Logical operation </param>
/// <param name="index"></param>
/// <returns> DocIdSet </returns>
- private DocIdSet GetDocIdSet(AtomicReaderContext context, int logic, int[] index)
+ private DocIdSet GetDocIdSet(AtomicReaderContext context, int logic, Span<int> index)
{
FixedBitSet result = InitialResult(context, logic, index);
for (; index[0] < chain.Length; index[0]++)
@@ -175,7 +176,7 @@ private DocIdSet GetDocIdSet(AtomicReaderContext context, int logic, int[] index
/// <param name="logic"> Logical operation </param>
/// <param name="index"></param>
/// <returns> DocIdSet </returns>
- private DocIdSet GetDocIdSet(AtomicReaderContext context, int[] logic, int[] index)
+ private DocIdSet GetDocIdSet(AtomicReaderContext context, int[] logic, Span<int> index)
{
if (logic.Length != chain.Length)
{
diff --git a/src/Lucene.Net.Tests/Support/Text/TestEncodingExtensions.cs b/src/Lucene.Net.Tests/Support/Text/TestEncodingExtensions.cs
index 55123917e..3fcdd8b44 100644
--- a/src/Lucene.Net.Tests/Support/Text/TestEncodingExtensions.cs
+++ b/src/Lucene.Net.Tests/Support/Text/TestEncodingExtensions.cs
@@ -3,6 +3,10 @@
using NUnit.Framework;
using System.Text;
+#if !FEATURE_ENCODING_GETSTRING_READONLYSPAN
+using System;
+#endif
+
namespace Lucene.Net.Support.Text
{
/*
@@ -38,5 +42,16 @@ public void TestWithDecoderExceptionFallback()
_ = newEncoding.GetString(new byte[] { 0xF0 });
});
}
+
+#if !FEATURE_ENCODING_GETSTRING_READONLYSPAN
+ [Test, LuceneNetSpecific]
+ public void TestGetString_ReadOnlySpan()
+ {
+ Encoding encoding = Encoding.UTF8;
+ Span<byte> bytes = stackalloc byte[] { 0x40 };
+ string s = encoding.GetString(bytes);
+ Assert.AreEqual("@", s);
+ }
+#endif
}
}
diff --git a/src/Lucene.Net/Support/Text/EncodingExtensions.cs b/src/Lucene.Net/Support/Text/EncodingExtensions.cs
index 5e1c3574c..fdbe1c0f4 100644
--- a/src/Lucene.Net/Support/Text/EncodingExtensions.cs
+++ b/src/Lucene.Net/Support/Text/EncodingExtensions.cs
@@ -1,5 +1,10 @@
using System.Collections.Concurrent;
using System.Text;
+
+#if !FEATURE_ENCODING_GETSTRING_READONLYSPAN
+using System;
+#endif
+
#nullable enable
namespace Lucene.Net.Support.Text
@@ -54,5 +59,15 @@ public static Encoding WithDecoderExceptionFallback(this Encoding encoding)
return newEncoding;
});
}
+
+#if !FEATURE_ENCODING_GETSTRING_READONLYSPAN
+ public static unsafe string GetString(this Encoding encoding, ReadOnlySpan<byte> bytes)
+ {
+ fixed (byte* bytesPtr = bytes)
+ {
+ return encoding.GetString(bytesPtr, bytes.Length);
+ }
+ }
+#endif
}
}