You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2020/02/19 15:15:51 UTC
[incubator-doris] branch master updated: [Alter] Add more schema
change to varchar type (#2777)
This is an automated email from the ASF dual-hosted git repository.
lichaoyong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new cc0d412 [Alter] Add more schema change to varchar type (#2777)
cc0d412 is described below
commit cc0d41277c8879691e6743663fbfe4def8dd0042
Author: WingC <10...@qq.com>
AuthorDate: Wed Feb 19 09:14:43 2020 -0600
[Alter] Add more schema change to varchar type (#2777)
---
be/src/olap/types.h | 28 +++++++-
be/test/olap/schema_change_test.cpp | 77 +++++++++++++++++++++-
.../sql-statements/Data Definition/ALTER TABLE.md | 1 +
.../Data Definition/ALTER TABLE_EN.md | 1 +
.../java/org/apache/doris/catalog/ColumnType.java | 13 +++-
5 files changed, 116 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index e2436db..09dd37d 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -477,6 +477,12 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_FLOAT> : public BaseFieldtypeTraits<OLAP_
*reinterpret_cast<CppType*>(buf) = value;
return OLAP_SUCCESS;
}
+ static std::string to_string(const void* src) {
+ char buf[1024] = {'\0'};
+ int length = FloatToBuffer(*reinterpret_cast<const CppType *>(src), MAX_FLOAT_STR_LENGTH, buf);
+ DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast<const CppType *>(src);
+ return std::string(buf);
+ }
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_float_from_varchar<CppType>(dest, src);
@@ -497,7 +503,8 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DOUBLE> : public BaseFieldtypeTraits<OLAP
}
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
- snprintf(buf, sizeof(buf), "%.10f", *reinterpret_cast<const CppType*>(src));
+ int length = DoubleToBuffer(*reinterpret_cast<const CppType *>(src), MAX_DOUBLE_STR_LENGTH, buf);
+ DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast<const CppType *>(src);
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
@@ -770,6 +777,25 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_VARCHAR> : public FieldTypeTraits<OLAP_FI
return OLAP_SUCCESS;
}
+ static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
+ if (src_type->type() == OLAP_FIELD_TYPE_TINYINT
+ || src_type->type() == OLAP_FIELD_TYPE_SMALLINT
+ || src_type->type() == OLAP_FIELD_TYPE_INT
+ || src_type->type() == OLAP_FIELD_TYPE_BIGINT
+ || src_type->type() == OLAP_FIELD_TYPE_LARGEINT
+ || src_type->type() == OLAP_FIELD_TYPE_FLOAT
+ || src_type->type() == OLAP_FIELD_TYPE_DOUBLE
+ || src_type->type() == OLAP_FIELD_TYPE_DECIMAL) {
+ auto result = src_type->to_string(src);
+ auto slice = reinterpret_cast<Slice*>(dest);
+ slice->data = reinterpret_cast<char*>(mem_pool->allocate(result.size()));
+ memcpy(slice->data, result.c_str(), result.size());
+ slice->size = result.size();
+ return OLAP_SUCCESS;
+ }
+ return OLAP_ERR_INVALID_SCHEMA;
+ }
+
static void set_to_min(void* buf) {
auto slice = reinterpret_cast<Slice*>(buf);
slice->size = 0;
diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp
index cf9e68a..0d2b931 100644
--- a/be/test/olap/schema_change_test.cpp
+++ b/be/test/olap/schema_change_test.cpp
@@ -212,7 +212,48 @@ public:
ASSERT_EQ(_column_writer->create_row_index_entry(), OLAP_SUCCESS);
}
- void test_convert_from_varchar(std::string type_name, int type_size, const std::string& value, OLAPStatus expected_st) {
+ template <typename T>
+ void test_convert_to_varchar(const std::string& type_name, int type_size, T val, const std::string& expected_val, OLAPStatus expected_st) {
+ TabletSchema src_tablet_schema;
+ SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &src_tablet_schema);
+ CreateColumnWriter(src_tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(src_tablet_schema);
+ RowBlock block(&src_tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+ write_row.set_field_content(0, reinterpret_cast<char*>(&val), _mem_pool.get());
+ block.set_row(0, write_row);
+ block.finalize(1);
+ ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
+ ColumnDataHeaderMessage header;
+ ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);
+ helper.close();
+
+ TabletSchema dst_tablet_schema;
+ SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &dst_tablet_schema);
+ CreateColumnReader(src_tablet_schema);
+ RowCursor read_row;
+ read_row.init(dst_tablet_schema);
+
+ _col_vector.reset(new ColumnVector());
+ ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS);
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ auto st = read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ ASSERT_EQ(st, expected_st);
+ if (st == OLAP_SUCCESS) {
+ std::string dst_str = read_row.column_schema(0)->to_string(read_row.cell_ptr(0));
+ ASSERT_TRUE(dst_str.compare(0, expected_val.size(), expected_val) == 0);
+ }
+
+ TypeInfo* tp = get_type_info(OLAP_FIELD_TYPE_HLL);
+ st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get());
+ ASSERT_EQ(st, OLAP_ERR_INVALID_SCHEMA);
+ }
+
+ void test_convert_from_varchar(const std::string& type_name, int type_size, const std::string& value, OLAPStatus expected_st) {
TabletSchema tablet_schema;
SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema);
CreateColumnWriter(tablet_schema);
@@ -230,8 +271,8 @@ public:
ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
ColumnDataHeaderMessage header;
ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);
-
helper.close();
+
TabletSchema converted_tablet_schema;
SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &converted_tablet_schema);
CreateColumnReader(tablet_schema);
@@ -560,6 +601,38 @@ TEST_F(TestColumn, ConvertVarcharToDouble) {
"1797690000000000063230304921389426434930330364336853362154109832891264341489062899406152996321966094455338163203127744334848599000464911410516510916727344709727599413825823048028128827530592629736371829425359826368844446113768685826367454055532068818593409163400929532301499014067384276511218551077374242324480.0000000000", OLAP_ERR_INVALID_SCHEMA);
}
+TEST_F(TestColumn, ConvertTinyIntToVarchar) {
+ test_convert_to_varchar<int8_t>("TINYINT", 1, 127, "127", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertSmallIntToVarchar) {
+ test_convert_to_varchar<int16_t>("SMALLINT", 2, 32767, "32767", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertIntToVarchar) {
+ test_convert_to_varchar<int32_t>("INT", 4, 2147483647, "2147483647", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertBigIntToVarchar) {
+ test_convert_to_varchar<int64_t>("BIGINT", 8, 9223372036854775807, "9223372036854775807", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertLargeIntToVarchar) {
+ test_convert_to_varchar<int128_t>("LARGEINT", 16, 1701411834604690, "1701411834604690", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertFloatToVarchar) {
+ test_convert_to_varchar<float>("FLOAT", 4, 3.40282e+38, "3.40282e+38", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertDoubleToVarchar) {
+ test_convert_to_varchar<double>("DOUBLE", 8, 123.456, "123.456", OLAP_SUCCESS);
+}
+
+TEST_F(TestColumn, ConvertDecimalToVarchar) {
+ decimal12_t val(456, 789000000);
+ test_convert_to_varchar<decimal12_t>("Decimal", 12, val, "456.789000000", OLAP_SUCCESS);
+}
}
int main(int argc, char** argv) {
diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md
index 3d2befd..b8974f6 100644
--- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md
+++ b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md
@@ -144,6 +144,7 @@ under the License.
4) 分区列不能做任何修改
5) 目前支持以下类型的转换（精度损失由用户保证）
TINYINT/SMALLINT/INT/BIGINT 转换成 TINYINT/SMALLINT/INT/BIGINT/DOUBLE。
+ TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL 转换成 VARCHAR
LARGEINT 转换成 DOUBLE
VARCHAR 支持修改最大长度
VARCHAR 转换成 TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE
diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md
index 2142a6c..bfbed30 100644
--- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md
+++ b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md
@@ -145,6 +145,7 @@ under the License.
4) The partition column cannot be modified
5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user)
TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE.
+ TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL is converted to VARCHAR
Convert LARGEINT to DOUBLE
VARCHAR supports modification of maximum length
Convert VARCHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE.
diff --git a/fe/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/src/main/java/org/apache/doris/catalog/ColumnType.java
index 7866213..557a6f5 100644
--- a/fe/src/main/java/org/apache/doris/catalog/ColumnType.java
+++ b/fe/src/main/java/org/apache/doris/catalog/ColumnType.java
@@ -47,19 +47,30 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.INT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DATE.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
+
+ schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
+
+ schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
+
+ schemaChangeMatrix[PrimitiveType.DOUBLE.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.CHAR.ordinal()] = true;
@@ -74,8 +85,8 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATE.ordinal()] = true;
+
schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIME.ordinal()] = true;
- schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
}
static boolean isSchemaChangeAllowed(Type lhs, Type rhs) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org