You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@doris.apache.org by GitBox <gi...@apache.org> on 2018/11/27 08:34:24 UTC
[GitHub] imay closed pull request #357: Remove unused row-oriented format
flags
imay closed pull request #357: Remove unused row-oriented format flags
URL: https://github.com/apache/incubator-doris/pull/357
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index 59bbf18b..2d63a3cf 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -348,8 +348,7 @@ OLAPStatus BaseCompaction::_do_base_compaction(VersionHash new_base_version_hash
uint64_t merged_rows = 0;
uint64_t filted_rows = 0;
OLAPStatus res = OLAP_SUCCESS;
- if (_table->data_file_type() == OLAP_DATA_FILE
- || _table->data_file_type() == COLUMN_ORIENTED_FILE) {
+ if (_table->data_file_type() == COLUMN_ORIENTED_FILE) {
_table->obtain_header_rdlock();
_table->release_header_lock();
diff --git a/be/src/olap/data_writer.cpp b/be/src/olap/data_writer.cpp
index a2a5ebca..257c6be3 100644
--- a/be/src/olap/data_writer.cpp
+++ b/be/src/olap/data_writer.cpp
@@ -51,7 +51,7 @@ ColumnDataWriter::ColumnDataWriter(OLAPTablePtr table, Rowset* index, bool is_pu
_segment_writer(NULL),
_num_rows(0),
_block_id(0),
- _max_segment_size(OLAP_MAX_SEGMENT_FILE_SIZE),
+ _max_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE),
_segment(0),
_all_num_rows(0),
_new_segment_created(false)
diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h
index c471dde1..24c1ec62 100644
--- a/be/src/olap/olap_define.h
+++ b/be/src/olap/olap_define.h
@@ -34,8 +34,6 @@ static const uint32_t OLAP_MAX_PATH_LEN = 512;
static const uint32_t OLAP_DEFAULT_MAX_PACKED_ROW_BLOCK_SIZE = 1024 * 1024 * 20;
// 每个row block压缩前的最大长度，也就是buf的最大长度
static const uint32_t OLAP_DEFAULT_MAX_UNPACKED_ROW_BLOCK_SIZE = 1024 * 1024 * 100;
-// 块大小使用uint32_t保存, 这里定义为2G,
-static const uint32_t OLAP_MAX_SEGMENT_FILE_SIZE = 2147483648;
// 列存储文件的块大小,由于可能会被全部载入内存,所以需要严格控制大小, 这里定义为256MB
static const uint32_t OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE = 268435456;
// 在列存储文件中, 数据分块压缩, 每个块的默认压缩前的大小
diff --git a/be/src/olap/olap_engine.cpp b/be/src/olap/olap_engine.cpp
index a513ec06..bb4f8afa 100644
--- a/be/src/olap/olap_engine.cpp
+++ b/be/src/olap/olap_engine.cpp
@@ -2009,15 +2009,11 @@ OLAPStatus OLAPEngine::_create_new_table_header(
} else {
header->set_keys_type(KeysType::AGG_KEYS);
}
- if (request.tablet_schema.storage_type == TStorageType::COLUMN) {
- header->set_data_file_type(COLUMN_ORIENTED_FILE);
- header->set_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE);
- header->set_num_rows_per_data_block(config::default_num_rows_per_column_file_block);
- } else {
- header->set_data_file_type(OLAP_DATA_FILE);
- header->set_segment_size(OLAP_MAX_SEGMENT_FILE_SIZE);
- header->set_num_rows_per_data_block(config::default_num_rows_per_data_block);
- }
+ DCHECK(request.tablet_schema.storage_type == TStorageType::COLUMN);
+ header->set_data_file_type(COLUMN_ORIENTED_FILE);
+ header->set_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE);
+ header->set_num_rows_per_data_block(config::default_num_rows_per_column_file_block);
+
// set column information
uint32_t i = 0;
uint32_t key_count = 0;
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 54d35170..9be1d1ee 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -1780,7 +1780,7 @@ OLAPStatus SchemaChangeHandler::schema_version_convert(
dest_olap_table,
rb_changer,
memory_limitation * 1024 * 1024 * 1024);
- } else if (true == sc_directly || src_olap_table->data_file_type() == OLAP_DATA_FILE) {
+ } else if (true == sc_directly) {
OLAP_LOG_INFO("doing schema change directly.");
sc_procedure = new(nothrow) SchemaChangeDirectly(
dest_olap_table, rb_changer);
@@ -2000,8 +2000,7 @@ OLAPStatus SchemaChangeHandler::_alter_table(SchemaChangeParams* sc_params) {
sc_params->new_olap_table,
rb_changer,
memory_limitation * 1024 * 1024 * 1024);
- } else if (true == sc_directly
- || sc_params->ref_olap_table->data_file_type() == OLAP_DATA_FILE) {
+ } else if (true == sc_directly) {
OLAP_LOG_INFO("doing schema change directly.");
sc_procedure = new(nothrow) SchemaChangeDirectly(
sc_params->new_olap_table, rb_changer);
diff --git a/be/test/olap/row_block_test.cpp b/be/test/olap/row_block_test.cpp
index c56944cb..04961dbc 100644
--- a/be/test/olap/row_block_test.cpp
+++ b/be/test/olap/row_block_test.cpp
@@ -84,7 +84,7 @@ TEST_F(TestRowBlock, init) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -96,12 +96,12 @@ TEST_F(TestRowBlock, init) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = false;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
// num_rows * (num_nullbytes + bigint + char + varchar)
- ASSERT_EQ(1024 * (8 + 10 + (4 + 20)), block.buf_len());
+ ASSERT_EQ(1024 * (3 + 8 + 10 + (4 + 20)), block.buf_len());
}
}
@@ -143,7 +143,7 @@ TEST_F(TestRowBlock, write_and_read) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -259,7 +259,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = false;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -375,7 +375,7 @@ TEST_F(TestRowBlock, compress_failed) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -452,7 +452,7 @@ TEST_F(TestRowBlock, decompress_failed) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -552,7 +552,7 @@ TEST_F(TestRowBlock, find_row) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -714,7 +714,7 @@ TEST_F(TestRowBlock, clear) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
@@ -764,7 +764,7 @@ TEST_F(TestRowBlock, pos_limit) {
RowBlock block(fields);
RowBlockInfo block_info;
block_info.row_num = 1024;
- block_info.data_file_type = OLAP_DATA_FILE;
+ block_info.data_file_type = COLUMN_ORIENTED_FILE;
block_info.null_supported = true;
auto res = block.init(block_info);
ASSERT_EQ(OLAP_SUCCESS, res);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index e8674f80..598f5cf4 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -96,7 +96,7 @@ message SchemaChangeStatusMessage {
}
enum DataFileType {
- OLAP_DATA_FILE = 0;
+ OLAP_DATA_FILE = 0; //Deprecated. Only columnar-wise format is supported.
COLUMN_ORIENTED_FILE = 1;
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@doris.apache.org
For additional commands, e-mail: dev-help@doris.apache.org