You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/10/16 15:57:15 UTC
[doris] branch branch-1.1-lts updated: [fix](jsonreader) teach jsonreader to release memory (#13336) (#13394)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new abceed6f5d [fix](jsonreader) teach jsonreader to release memory (#13336) (#13394)
abceed6f5d is described below
commit abceed6f5d6fef9471415ad82f5b04a884074987
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Sun Oct 16 23:57:10 2022 +0800
[fix](jsonreader) teach jsonreader to release memory (#13336) (#13394)
Allocator of rapidjson does not release memory, this fix use allocator with local buffer and call Clear to release memory allocated beyond local buffer.
---
be/src/exec/json_scanner.cpp | 5 +++++
be/src/exec/json_scanner.h | 10 +++++++++-
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/json_scanner.cpp b/be/src/exec/json_scanner.cpp
index d651545806..f2273ffca4 100644
--- a/be/src/exec/json_scanner.cpp
+++ b/be/src/exec/json_scanner.cpp
@@ -296,6 +296,9 @@ JsonReader::JsonReader(RuntimeState* state, ScannerCounter* counter, RuntimeProf
_strip_outer_array(strip_outer_array),
_num_as_string(num_as_string),
_fuzzy_parse(fuzzy_parse),
+ _value_allocator(_value_buffer, sizeof(_value_buffer)),
+ _parse_allocator(_parse_buffer, sizeof(_parse_buffer)),
+ _origin_json_doc(&_value_allocator, sizeof(_parse_buffer), &_parse_allocator),
_json_doc(nullptr),
_scanner_eof(scanner_eof) {
_bytes_read_counter = ADD_COUNTER(_profile, "BytesRead", TUnit::BYTES);
@@ -386,6 +389,8 @@ Status JsonReader::_parse_json_doc(size_t* size, bool* eof) {
return Status::OK();
}
+ // clear memory here.
+ _origin_json_doc.GetAllocator().Clear();
bool has_parse_error = false;
// parse jsondata to JsonDoc
diff --git a/be/src/exec/json_scanner.h b/be/src/exec/json_scanner.h
index 1a489a5ed8..846d3a24a4 100644
--- a/be/src/exec/json_scanner.h
+++ b/be/src/exec/json_scanner.h
@@ -181,7 +181,15 @@ private:
std::vector<std::vector<JsonPath>> _parsed_jsonpaths;
std::vector<JsonPath> _parsed_json_root;
- rapidjson::Document _origin_json_doc; // origin json document object from parsed json string
+ char _value_buffer[4 * 1024 * 1024];
+ char _parse_buffer[512 * 1024];
+
+ typedef rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::MemoryPoolAllocator<>,
+ rapidjson::MemoryPoolAllocator<>>
+ Document;
+ rapidjson::MemoryPoolAllocator<> _value_allocator;
+ rapidjson::MemoryPoolAllocator<> _parse_allocator;
+ Document _origin_json_doc; // origin json document object from parsed json string
rapidjson::Value* _json_doc; // _json_doc equals _final_json_doc iff not set `json_root`
std::unordered_map<std::string, int> _name_map;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org