You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2021/09/01 08:00:13 UTC
[incubator-doris] branch master updated: [BUG] fix bugs with string
type (#6538)
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9f7d4cf [BUG] fix bugs with string type (#6538)
9f7d4cf is described below
commit 9f7d4cf741ea92db96a182c918f22ec404507a19
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Wed Sep 1 15:59:55 2021 +0800
[BUG] fix bugs with string type (#6538)
* fix bugs with string type
1. not support string with agg type min/max
2. agg_update with large string may coredump
3. stringval with large string may coredump
4. not support string as partition key
---
be/src/olap/field.h | 9 +++++++++
be/src/runtime/free_pool.hpp | 12 ++++++------
be/src/runtime/row_batch.cpp | 8 ++++----
.../src/main/java/org/apache/doris/analysis/LiteralExpr.java | 1 +
.../main/java/org/apache/doris/catalog/AggregateType.java | 2 ++
.../java/org/apache/doris/catalog/ListPartitionInfo.java | 2 +-
.../src/main/java/org/apache/doris/catalog/PartitionKey.java | 1 +
.../org/apache/doris/analysis/CreateRoutineLoadStmtTest.java | 8 ++++++++
8 files changed, 32 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/field.h b/be/src/olap/field.h
index 60cd4c1..f2f1129 100644
--- a/be/src/olap/field.h
+++ b/be/src/olap/field.h
@@ -85,6 +85,15 @@ public:
inline void agg_update(RowCursorCell* dest, const RowCursorCell& src,
MemPool* mem_pool = nullptr) const {
+ if (type() == OLAP_FIELD_TYPE_STRING && mem_pool == nullptr) {
+ auto dst_slice = reinterpret_cast<Slice*>(dest->mutable_cell_ptr());
+ auto src_slice = reinterpret_cast<const Slice*>(src.cell_ptr());
+ if (dst_slice->size < src_slice->size) {
+ *_long_text_buf = static_cast<char*>(realloc(*_long_text_buf, src_slice->size));
+ dst_slice->data = *_long_text_buf;
+ dst_slice->size = src_slice->size;
+ }
+ }
_agg_info->update(dest, src, mem_pool);
}
diff --git a/be/src/runtime/free_pool.hpp b/be/src/runtime/free_pool.hpp
old mode 100755
new mode 100644
index 0f4aad3..379d254
--- a/be/src/runtime/free_pool.hpp
+++ b/be/src/runtime/free_pool.hpp
@@ -20,7 +20,9 @@
#include <stdio.h>
#include <string.h>
+
#include <string>
+
#include "common/logging.h"
#include "runtime/mem_pool.h"
#include "util/bit_util.h"
@@ -44,9 +46,7 @@ class FreePool {
public:
// C'tor, initializes the FreePool to be empty. All allocations come from the
// 'mem_pool'.
- FreePool(MemPool* mem_pool) : _mem_pool(mem_pool) {
- memset(&_lists, 0, sizeof(_lists));
- }
+ FreePool(MemPool* mem_pool) : _mem_pool(mem_pool) { memset(&_lists, 0, sizeof(_lists)); }
virtual ~FreePool() {}
@@ -65,9 +65,9 @@ public:
if (allocation == NULL) {
// There wasn't an existing allocation of the right size, allocate a new one.
- size = 1 << free_list_idx;
+ size = 1L << free_list_idx;
allocation = reinterpret_cast<FreeListNode*>(
- _mem_pool->allocate(size + sizeof(FreeListNode)));
+ _mem_pool->allocate(size + sizeof(FreeListNode)));
} else {
// Remove this allocation from the list.
_lists[free_list_idx].next = allocation->next;
@@ -158,6 +158,6 @@ private:
FreeListNode _lists[NUM_LISTS];
};
-}
+} // namespace doris
#endif
diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp
index bf2ee39..41b303d 100644
--- a/be/src/runtime/row_batch.cpp
+++ b/be/src/runtime/row_batch.cpp
@@ -20,14 +20,14 @@
#include <snappy/snappy.h>
#include <stdint.h> // for intptr_t
+#include "gen_cpp/Data_types.h"
+#include "gen_cpp/data.pb.h"
#include "runtime/buffered_tuple_stream2.inline.h"
+#include "runtime/collection_value.h"
#include "runtime/exec_env.h"
#include "runtime/runtime_state.h"
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
-#include "gen_cpp/Data_types.h"
-#include "gen_cpp/data.pb.h"
-#include "runtime/collection_value.h"
//#include "vec/columns/column_vector.h"
//#include "vec/core/block.h"
@@ -478,7 +478,7 @@ int RowBatch::serialize(PRowBatch* output_batch) {
if (config::compress_rowbatches && size > 0) {
// Try compressing tuple_data to _compression_scratch, swap if compressed data is
// smaller
- int max_compressed_size = snappy::MaxCompressedLength(size);
+ uint32_t max_compressed_size = snappy::MaxCompressedLength(size);
if (_compression_scratch.size() < max_compressed_size) {
_compression_scratch.resize(max_compressed_size);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
index 1665a4b..6e435cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
@@ -72,6 +72,7 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
case CHAR:
case VARCHAR:
case HLL:
+ case STRING:
literalExpr = new StringLiteral(value);
break;
case DATE:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java
index e7f253e..bcd9e50 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java
@@ -64,6 +64,7 @@ public enum AggregateType {
primitiveTypeList.add(PrimitiveType.DATETIME);
primitiveTypeList.add(PrimitiveType.CHAR);
primitiveTypeList.add(PrimitiveType.VARCHAR);
+ primitiveTypeList.add(PrimitiveType.STRING);
compatibilityMap.put(MIN, EnumSet.copyOf(primitiveTypeList));
primitiveTypeList.clear();
@@ -79,6 +80,7 @@ public enum AggregateType {
primitiveTypeList.add(PrimitiveType.DATETIME);
primitiveTypeList.add(PrimitiveType.CHAR);
primitiveTypeList.add(PrimitiveType.VARCHAR);
+ primitiveTypeList.add(PrimitiveType.STRING);
compatibilityMap.put(MAX, EnumSet.copyOf(primitiveTypeList));
primitiveTypeList.clear();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java
index e53404d..201c89d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java
@@ -75,7 +75,7 @@ public class ListPartitionInfo extends PartitionInfo{
partitionKeys.add(partitionKey);
}
} catch (AnalysisException e) {
- throw new DdlException("Invalid list value format： " + e.getMessage());
+ throw new DdlException("Invalid list value format: " + e.getMessage());
}
return new ListPartitionItem(partitionKeys);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
index e0aec61..93b2ba1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
@@ -309,6 +309,7 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
break;
case CHAR:
case VARCHAR:
+ case STRING:
literal = StringLiteral.read(in);
break;
case BOOLEAN:
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
index 8c3a1fa..1607733 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.load.routineload.LoadDataSourceType;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -54,6 +55,9 @@ public class CreateRoutineLoadStmtTest {
private Catalog catalog;
@Mocked
+ private ConnectContext ctx;
+
+ @Mocked
OlapTable table;
@Before
@@ -77,6 +81,10 @@ public class CreateRoutineLoadStmtTest {
table.hasDeleteSign();
minTimes = 0;
result = false;
+
+ ConnectContext.get();
+ minTimes = 0;
+ result = ctx;
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org