You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ad...@apache.org on 2023/04/27 06:15:07 UTC
[doris-thirdparty] branch clucene updated: [fix](analyzer) Fix memory leak in Standard Analyzer reuseTokenStream (#60)
This is an automated email from the ASF dual-hosted git repository.
adonisling pushed a commit to branch clucene
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene by this push:
new f0eb372 [fix](analyzer) Fix memory leak in Standard Analyzer reuseTokenStream (#60)
f0eb372 is described below
commit f0eb3726b65038e43b83e6f577782eb9c4043b80
Author: airborne12 <ai...@gmail.com>
AuthorDate: Thu Apr 27 14:15:02 2023 +0800
[fix](analyzer) Fix memory leak in Standard Analyzer reuseTokenStream (#60)
---
src/core/CLucene/analysis/AnalysisHeader.h | 4 ++--
src/core/CLucene/analysis/standard/StandardAnalyzer.cpp | 5 ++++-
src/core/CLucene/analysis/standard/StandardAnalyzer.h | 8 ++++++++
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/core/CLucene/analysis/AnalysisHeader.h b/src/core/CLucene/analysis/AnalysisHeader.h
index e3d572a..de19835 100644
--- a/src/core/CLucene/analysis/AnalysisHeader.h
+++ b/src/core/CLucene/analysis/AnalysisHeader.h
@@ -267,12 +267,12 @@ protected:
/** Used by Analyzers that implement reusableTokenStream
* to retrieve previously saved TokenStreams for re-use
* by the same thread. */
- TokenStream* getPreviousTokenStream();
+ virtual TokenStream* getPreviousTokenStream();
/** Used by Analyzers that implement reusableTokenStream
* to save a TokenStream for later re-use by the same
* thread. */
- void setPreviousTokenStream(TokenStream* obj);
+ virtual void setPreviousTokenStream(TokenStream* obj);
public:
/**
* Invoked before indexing a Field instance if
diff --git a/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp b/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp
index 3275c1b..d0383f2 100644
--- a/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp
+++ b/src/core/CLucene/analysis/standard/StandardAnalyzer.cpp
@@ -60,7 +60,10 @@ CL_NS_DEF2(analysis,standard)
StandardAnalyzer::~StandardAnalyzer(){
SavedStreams* t = reinterpret_cast<SavedStreams*>(this->getPreviousTokenStream());
- if (t) _CLDELETE(t->filteredTokenStream);
+ if (t) {
+ _CLDELETE(t->filteredTokenStream);
+ _CLDELETE(t);
+ }
_CLLDELETE(stopSet);
}
diff --git a/src/core/CLucene/analysis/standard/StandardAnalyzer.h b/src/core/CLucene/analysis/standard/StandardAnalyzer.h
index 85e476b..653ac85 100644
--- a/src/core/CLucene/analysis/standard/StandardAnalyzer.h
+++ b/src/core/CLucene/analysis/standard/StandardAnalyzer.h
@@ -22,6 +22,7 @@ CL_NS_DEF2(analysis,standard)
private:
CLTCSetList* stopSet;
int32_t maxTokenLength;
+ TokenStream* prev_tokenStream{};
class SavedStreams;
public:
@@ -46,6 +47,13 @@ CL_NS_DEF2(analysis,standard)
virtual ~StandardAnalyzer();
+
+ TokenStream* getPreviousTokenStream() override {
+ return prev_tokenStream;
+ }
+ void setPreviousTokenStream(TokenStream* obj) override {
+ prev_tokenStream = obj;
+ }
/**
* Constructs a StandardTokenizer filtered by a
* StandardFilter, a LowerCaseFilter and a StopFilter.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org