You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/10/19 05:42:46 UTC
[incubator-doris] branch branch-0.15 updated (8fa3bbf -> 259cc7b)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git.
from 8fa3bbf [Dependency] Upgrade thirdparty libs (#6766)
new e386b8b [Doc] Update compile docs add 0.15 build support. (#6850)
new 59ce135 [BUG] Key is 'True',Extra is 'NONE' when add rollup for DUP table (#6763)
new 0158442 [MemerySave] Change TabletSchema in tablet to reference to save mem (#6814)
new 171ee71 [SQL Cache] Add all view stmt as the suffix of cache sqlkey (#6832)
new 91fd17d [ARM64] Fix some problem when compiling on ARM64 platform (#6836)
new ba5a376 [BUG] Fix printing ReservationTrackerCounters cause BE crash when mem_limit is reached (#6849)
new dce0a1e [BUG] fix bug for schema schange (#6839)
new 6c42fcc [Doc] Add type BOOLEAN when enter 'help create table' in mysql client (#6852)
new 986bd1a [Bug] Fix bug that BE will crash when backup using S3 (#6855)
new 26e3a37 [UT] Fix ut bugs (#6862)
new afc7131 [Docs] Fix exporter document error (#6864)
new bcc310b Update materialized_view.md (#6867)
new ead82c0 [Doc] Add documents for MySQL Binlog Load (#6859)
new 259cc7b [ARM64] Fix some problem when compiling on ARM64 platform (#6836) (#6872)
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/CMakeLists.txt | 8 +-
be/src/exec/CMakeLists.txt | 3 +-
be/src/exec/broker_scanner.cpp | 13 +-
be/src/exec/hdfs_file_reader.cpp | 3 +-
be/src/exec/hdfs_file_reader.h | 2 +-
...ma_dummy_scanner.cpp => hdfs_reader_writer.cpp} | 37 +-
.../result_writer.h => exec/hdfs_reader_writer.h} | 48 +-
be/src/exec/hdfs_writer.cpp | 2 +-
be/src/exec/hdfs_writer.h | 2 +-
be/src/exec/parquet_scanner.cpp | 13 +-
be/src/glibc-compatibility/CMakeLists.txt | 4 +-
be/src/olap/base_tablet.cpp | 2 -
be/src/olap/base_tablet.h | 2 +-
be/src/olap/tablet.cpp | 6 +-
be/src/olap/tablet_meta.cpp | 31 +-
be/src/olap/tablet_meta.h | 13 +-
be/src/runtime/file_result_writer.cpp | 6 +-
be/src/runtime/mem_tracker.cpp | 23 +-
be/src/runtime/snapshot_loader.cpp | 6 +-
be/src/runtime/snapshot_loader.h | 4 +-
be/src/util/aes_util.cpp | 16 +-
be/src/util/vectorized-tool/lower_upper_impl.h | 4 +-
be/test/olap/test_data/header_without_inc_rs.txt | 1 +
docs/.vuepress/public/images/canal_store.png | Bin 0 -> 51780 bytes
docs/.vuepress/sidebar/en.js | 6 +
docs/.vuepress/sidebar/zh-CN.js | 6 +
docs/en/administrator-guide/export-manual.md | 3 +-
.../load-data/binlog-load-manual.md | 492 +++++++++++++++++++++
docs/en/installing/compilation-arm.md | 11 +-
docs/en/installing/compilation.md | 9 +-
.../sql-statements/Data Definition/CREATE TABLE.md | 9 +-
.../Data Manipulation/CREATE SYNC JOB.md | 165 +++++++
.../PAUSE SYNC JOB.md} | 27 +-
.../RESUME SYNC JOB.md} | 23 +-
.../SHOW SYNC JOB.md} | 32 +-
.../STOP SYNC JOB.md} | 21 +-
docs/zh-CN/administrator-guide/export-manual.md | 4 +-
.../load-data/binlog-load-manual.md | 470 ++++++++++++++++++++
.../zh-CN/administrator-guide/materialized_view.md | 2 +-
docs/zh-CN/installing/compilation-arm.md | 15 +-
docs/zh-CN/installing/compilation.md | 9 +-
.../sql-statements/Data Definition/CREATE TABLE.md | 9 +-
.../Data Manipulation/CREATE SYNC JOB.md | 166 +++++++
.../Data Manipulation/PAUSE SYNC JOB.md} | 27 +-
.../Data Manipulation/RESUME SYNC JOB.md} | 29 +-
.../SHOW SYNC JOB.md} | 29 +-
.../Data Manipulation/STOP SYNC JOB.md} | 27 +-
.../apache/doris/alter/SchemaChangeHandler.java | 4 +-
.../org/apache/doris/analysis/InlineViewRef.java | 8 +
.../main/java/org/apache/doris/catalog/Column.java | 4 +
.../main/java/org/apache/doris/qe/cache/Cache.java | 1 +
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 43 +-
.../org/apache/doris/qe/cache/PartitionCache.java | 7 +-
.../java/org/apache/doris/qe/cache/SqlCache.java | 6 +-
.../java/org/apache/doris/alter/AlterTest.java | 20 +
.../org/apache/doris/alter/RollupJobV2Test.java | 20 +
.../org/apache/doris/catalog/CatalogTestUtil.java | 66 ++-
thirdparty/CHANGELOG.md | 14 +-
thirdparty/download-thirdparty.sh | 12 +
thirdparty/patches/aws-c-cal-0.4.5.patch | 11 +
thirdparty/vars.sh | 8 +-
61 files changed, 1802 insertions(+), 262 deletions(-)
copy be/src/exec/{schema_scanner/schema_dummy_scanner.cpp => hdfs_reader_writer.cpp} (52%)
copy be/src/{runtime/result_writer.h => exec/hdfs_reader_writer.h} (53%)
create mode 100644 docs/.vuepress/public/images/canal_store.png
create mode 100644 docs/en/administrator-guide/load-data/binlog-load-manual.md
create mode 100644 docs/en/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md
copy docs/en/sql-reference/sql-statements/{Administration/UNINSTALL PLUGIN.md => Data Manipulation/PAUSE SYNC JOB.md} (71%)
copy docs/en/sql-reference/sql-statements/{Administration/SHOW PLUGINS.md => Data Manipulation/RESUME SYNC JOB.md} (72%)
copy docs/en/sql-reference/sql-statements/{Administration/UNINSTALL PLUGIN.md => Data Manipulation/SHOW SYNC JOB.md} (69%)
copy docs/en/sql-reference/sql-statements/{Data Definition/DROP INDEX.md => Data Manipulation/STOP SYNC JOB.md} (78%)
create mode 100644 docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md
create mode 100644 docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md
copy docs/zh-CN/sql-reference/{sql-functions/aggregate-functions/max.md => sql-statements/Data Manipulation/PAUSE SYNC JOB.md} (69%)
mode change 100755 => 100644
copy docs/zh-CN/sql-reference/{sql-functions/date-time-functions/day.md => sql-statements/Data Manipulation/RESUME SYNC JOB.md} (70%)
copy docs/zh-CN/sql-reference/sql-statements/{Account Management/DROP USER.md => Data Manipulation/SHOW SYNC JOB.md} (69%)
copy docs/zh-CN/sql-reference/{sql-functions/date-time-functions/now.md => sql-statements/Data Manipulation/STOP SYNC JOB.md} (75%)
create mode 100644 thirdparty/patches/aws-c-cal-0.4.5.patch
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 07/14: [BUG] fix bug for schema schange (#6839)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit dce0a1e0e0285078c1abce0ec45a5584847549be
Author: pengxiangyu <di...@163.com>
AuthorDate: Sun Oct 17 22:53:28 2021 +0800
[BUG] fix bug for schema schange (#6839)
This commit has an error:
#6791
when you only change the order of column, error will apear.
---
.../org/apache/doris/alter/SchemaChangeHandler.java | 4 ++--
.../test/java/org/apache/doris/alter/AlterTest.java | 20 ++++++++++++++++++++
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 313ad47..2e6ba17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1221,7 +1221,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Column alterColumn : alterSchema) {
if (alterColumn.nameEquals(partitionCol.getName(), true)) {
// 2.1 partition column cannot be modified
- if (needAlterColumns.contains(alterColumn)) {
+ if (needAlterColumns.contains(alterColumn) && !alterColumn.equals(partitionCol)) {
throw new DdlException("Can not modify partition column["
+ partitionCol.getName() + "]. index["
+ olapTable.getIndexNameById(alterIndexId) + "]");
@@ -1250,7 +1250,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Column alterColumn : alterSchema) {
if (alterColumn.nameEquals(distributionCol.getName(), true)) {
// 3.1 distribution column cannot be modified
- if (needAlterColumns.contains(alterColumn)) {
+ if (needAlterColumns.contains(alterColumn) && !alterColumn.equals(distributionCol)) {
throw new DdlException("Can not modify distribution column["
+ distributionCol.getName() + "]. index["
+ olapTable.getIndexNameById(alterIndexId) + "]");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
index a2a0667..92a05d9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
@@ -704,6 +704,26 @@ public class AlterTest {
}
+ @Test
+ public void testChangeOrder() throws Exception {
+ createTable("CREATE TABLE test.change_order\n" +
+ "(\n" +
+ " k1 date,\n" +
+ " k2 int,\n" +
+ " v1 int sum\n" +
+ ")\n" +
+ "PARTITION BY RANGE(k1)\n" +
+ "(\n" +
+ " PARTITION p1 values less than('2020-02-01'),\n" +
+ " PARTITION p2 values less than('2020-03-01')\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" +
+ "PROPERTIES('replication_num' = '1');");
+
+ String changeOrderStmt = "ALTER TABLE test.change_order ORDER BY (k2, k1, v1);;";
+ alterTable(changeOrderStmt, false);
+ }
+
private boolean checkAllTabletsExists(List<Long> tabletIds) {
TabletInvertedIndex invertedIndex = Catalog.getCurrentCatalog().getTabletInvertedIndex();
for (long tabletId : tabletIds) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 11/14: [Docs] Fix exporter document error (#6864)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit afc7131b15cbd534bd8125bc8331e4ca27a70b51
Author: wunan1210 <wu...@gmail.com>
AuthorDate: Tue Oct 19 10:24:08 2021 +0800
[Docs] Fix exporter document error (#6864)
* fix exporter document error
* update en doc
---
docs/en/administrator-guide/export-manual.md | 3 ++-
docs/zh-CN/administrator-guide/export-manual.md | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/docs/en/administrator-guide/export-manual.md b/docs/en/administrator-guide/export-manual.md
index 1d9e565..df2dbbb 100644
--- a/docs/en/administrator-guide/export-manual.md
+++ b/docs/en/administrator-guide/export-manual.md
@@ -108,13 +108,14 @@ PROPERTIES
(
"label"="mylabel",
"column_separator"=",",
+ "columns" = "col1,col2",
"exec_mem_limit"="2147483648",
"timeout" = "3600"
)
WITH BROKER "hdfs"
(
"username" = "user",
- "password" = "passwd",
+ "password" = "passwd"
);
```
diff --git a/docs/zh-CN/administrator-guide/export-manual.md b/docs/zh-CN/administrator-guide/export-manual.md
index 10925bc..89f3bbe 100644
--- a/docs/zh-CN/administrator-guide/export-manual.md
+++ b/docs/zh-CN/administrator-guide/export-manual.md
@@ -112,14 +112,14 @@ PROPERTIES
(
"label" = "mylabel",
"column_separator"=",",
- "columns":"col1,col2"
+ "columns" = "col1,col2",
"exec_mem_limit"="2147483648",
"timeout" = "3600"
)
WITH BROKER "hdfs"
(
"username" = "user",
- "password" = "passwd",
+ "password" = "passwd"
);
```
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 08/14: [Doc] Add type BOOLEAN when enter 'help
create table' in mysql client (#6852)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 6c42fccbe0946878a4a0eb5fb2358d7d669ab4da
Author: zhoubintao <35...@users.noreply.github.com>
AuthorDate: Sun Oct 17 22:54:12 2021 +0800
[Doc] Add type BOOLEAN when enter 'help create table' in mysql client (#6852)
some user do not know Doris support type boolean, they use TINYINT,
so i add type BOOLEAN when enter 'help create table' in mysql client.
currently, type BOOLEAN size is 1 byte, but the value of boolean column only in {0,1} ,
which waste some memory, and i want change it's implement to 1 bit in the future.
---
.../sql-reference/sql-statements/Data Definition/CREATE TABLE.md | 9 ++++++---
.../sql-reference/sql-statements/Data Definition/CREATE TABLE.md | 9 ++++++---
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
index b8275a2..707f19d 100644
--- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
+++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
@@ -52,6 +52,8 @@ Syntax:
col_name: Name of column
col_type: Type of column
```
+ BOOLEAN(1 Byte)
+ Range: {0,1}
TINYINT(1 Byte)
Range: -2^7 + 1 ~ 2^7 - 1
SMALLINT(2 Bytes)
@@ -338,13 +340,14 @@ Syntax:
```
CREATE TABLE example_db.table_hash
(
- k1 TINYINT,
- k2 DECIMAL(10, 2) DEFAULT "10.5",
+ k1 BOOLEAN,
+ k2 TINYINT,
+ k3 DECIMAL(10, 2) DEFAULT "10.5",
v1 CHAR(10) REPLACE,
v2 INT SUM
)
ENGINE=olap
- AGGREGATE KEY(k1, k2)
+ AGGREGATE KEY(k1, k2, k3)
COMMENT "my first doris table"
DISTRIBUTED BY HASH(k1) BUCKETS 32;
```
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
index 9623a5d..a99fd99 100644
--- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
@@ -54,6 +54,8 @@ under the License.
col_type：列类型
```
+ BOOLEAN（1字节）
+ 范围：{0,1}
TINYINT（1字节）
范围：-2^7 + 1 ~ 2^7 - 1
SMALLINT（2字节）
@@ -370,13 +372,14 @@ under the License.
```
CREATE TABLE example_db.table_hash
(
- k1 TINYINT,
- k2 DECIMAL(10, 2) DEFAULT "10.5",
+ k1 BOOLEAN,
+ k2 TINYINT,
+ k3 DECIMAL(10, 2) DEFAULT "10.5",
v1 CHAR(10) REPLACE,
v2 INT SUM
)
ENGINE=olap
- AGGREGATE KEY(k1, k2)
+ AGGREGATE KEY(k1, k2, k3)
COMMENT "my first doris table"
DISTRIBUTED BY HASH(k1) BUCKETS 32;
```
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 03/14: [MemerySave] Change TabletSchema in tablet
to reference to save mem (#6814)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 0158442c8645dde8d4e79a6265b45ff1ad85750d
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Sat Oct 16 21:54:32 2021 +0800
[MemerySave] Change TabletSchema in tablet to reference to save mem (#6814)
Change TabletSchema in tablet to reference to save memory
---
be/src/olap/base_tablet.cpp | 2 --
be/src/olap/base_tablet.h | 2 +-
be/src/olap/tablet.cpp | 6 +-----
be/src/olap/tablet_meta.cpp | 31 +++++++++++++++++++++++++------
be/src/olap/tablet_meta.h | 11 ++++++++---
5 files changed, 35 insertions(+), 17 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 35b9d7e..0f634ba 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -31,8 +31,6 @@ extern MetricPrototype METRIC_query_scan_count;
BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir)
: _state(tablet_meta->tablet_state()),
_tablet_meta(tablet_meta),
- // TODO: Think we really copy tablet schema here, which cost double mem
- // cost in TabletManager
_schema(tablet_meta->tablet_schema()),
_data_dir(data_dir) {
_gen_tablet_path();
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 9d0e331..3e1386b 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -70,7 +70,7 @@ protected:
protected:
TabletState _state;
TabletMetaSharedPtr _tablet_meta;
- TabletSchema _schema;
+ const TabletSchema& _schema;
DataDir* _data_dir;
std::string _tablet_path;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 3d22614..b6dc448 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -136,9 +136,6 @@ void Tablet::save_meta() {
auto res = _tablet_meta->save_meta(_data_dir);
CHECK_EQ(res, OLAP_SUCCESS) << "fail to save tablet_meta. res=" << res
<< ", root=" << _data_dir->path();
- // User could directly update tablet schema by _tablet_meta,
- // So we need to refetch schema again
- _schema = _tablet_meta->tablet_schema();
}
OLAPStatus Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>& rowsets_to_clone,
@@ -149,8 +146,7 @@ OLAPStatus Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>& ro
OLAPStatus res = OLAP_SUCCESS;
do {
// load new local tablet_meta to operate on
- TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
- generate_tablet_meta_copy_unlocked(new_tablet_meta);
+ TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta(*_tablet_meta));
// delete versions from new local tablet_meta
for (const Version& version : versions_to_delete) {
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 824cdaa..564ced8 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -46,14 +46,14 @@ OLAPStatus TabletMeta::create(const TCreateTabletReq& request, const TabletUid&
return OLAP_SUCCESS;
}
-TabletMeta::TabletMeta() : _tablet_uid(0, 0) {}
+TabletMeta::TabletMeta() : _tablet_uid(0, 0), _schema(new TabletSchema) {}
TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id,
int32_t schema_hash, uint64_t shard_id, const TTabletSchema& tablet_schema,
uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletUid tablet_uid, TTabletType::type tabletType)
- : _tablet_uid(0, 0), _preferred_rowset_type(ALPHA_ROWSET) {
+ : _tablet_uid(0, 0), _preferred_rowset_type(BETA_ROWSET), _schema(new TabletSchema) {
TabletMetaPB tablet_meta_pb;
tablet_meta_pb.set_table_id(table_id);
tablet_meta_pb.set_partition_id(partition_id);
@@ -139,6 +139,25 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
init_from_pb(tablet_meta_pb);
}
+TabletMeta::TabletMeta(const TabletMeta& b)
+ : _table_id(b._table_id),
+ _partition_id(b._partition_id),
+ _tablet_id(b._tablet_id),
+ _schema_hash(b._schema_hash),
+ _shard_id(b._shard_id),
+ _creation_time(b._creation_time),
+ _cumulative_layer_point(b._cumulative_layer_point),
+ _tablet_uid(b._tablet_uid),
+ _tablet_type(b._tablet_type),
+ _tablet_state(b._tablet_state),
+ _schema(b._schema),
+ _in_restore_mode(b._in_restore_mode),
+ _rs_metas(b._rs_metas),
+ _stale_rs_metas(b._stale_rs_metas),
+ _del_pred_array(b._del_pred_array),
+ _preferred_rowset_type(b._preferred_rowset_type) {
+}
+
void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
ColumnPB* column) {
column->set_unique_id(unique_id);
@@ -350,7 +369,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
}
// init _schema
- _schema.init_from_pb(tablet_meta_pb.schema());
+ _schema->init_from_pb(tablet_meta_pb.schema());
// init _rs_metas
for (auto& it : tablet_meta_pb.rs_metas()) {
@@ -411,7 +430,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
for (auto rs : _stale_rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_stale_rs_metas());
}
- _schema.to_schema_pb(tablet_meta_pb->mutable_schema());
+ _schema->to_schema_pb(tablet_meta_pb->mutable_schema());
tablet_meta_pb->set_in_restore_mode(in_restore_mode());
@@ -423,7 +442,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
uint32_t TabletMeta::mem_size() const {
auto size = sizeof(TabletMeta);
- size += _schema.mem_size();
+ size += _schema->mem_size();
return size;
}
@@ -625,7 +644,7 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) {
if (a._tablet_uid != b._tablet_uid) return false;
if (a._tablet_type != b._tablet_type) return false;
if (a._tablet_state != b._tablet_state) return false;
- if (a._schema != b._schema) return false;
+ if (*a._schema != *b._schema) return false;
if (a._rs_metas.size() != b._rs_metas.size()) return false;
for (int i = 0; i < a._rs_metas.size(); ++i) {
if (a._rs_metas[i] != b._rs_metas[i]) return false;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 9f3ba46..8995e96 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -82,6 +82,9 @@ public:
uint64_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletUid tablet_uid, TTabletType::type tabletType);
+ // If need add a filed in TableMeta, filed init copy in copy construct function
+ TabletMeta(const TabletMeta& tablet_meta);
+ TabletMeta(TabletMeta&& tablet_meta) = delete;
// Function create_from_file is used to be compatible with previous tablet_meta.
// Previous tablet_meta is a physical file in tablet dir, which is not stored in rocksdb.
@@ -184,7 +187,9 @@ private:
TabletTypePB _tablet_type = TabletTypePB::TABLET_TYPE_DISK;
TabletState _tablet_state = TABLET_NOTREADY;
- TabletSchema _schema;
+ // the reference of _schema may use in tablet, so here need keep
+ // the lifetime of tablemeta and _schema is same with tablet
+ std::shared_ptr<TabletSchema> _schema;
std::vector<RowsetMetaSharedPtr> _rs_metas;
// This variable _stale_rs_metas is used to record these rowsets‘ meta which are be compacted.
@@ -282,11 +287,11 @@ inline void TabletMeta::set_in_restore_mode(bool in_restore_mode) {
}
inline const TabletSchema& TabletMeta::tablet_schema() const {
- return _schema;
+ return *_schema;
}
inline TabletSchema* TabletMeta::mutable_tablet_schema() {
- return &_schema;
+ return _schema.get();
}
inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas() const {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 12/14: Update materialized_view.md (#6867)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit bcc310bc5b53e07ee6d431a467d0830f6c0ba4a5
Author: MHBoy <ch...@163.com>
AuthorDate: Tue Oct 19 10:24:38 2021 +0800
Update materialized_view.md (#6867)
---
docs/zh-CN/administrator-guide/materialized_view.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/zh-CN/administrator-guide/materialized_view.md b/docs/zh-CN/administrator-guide/materialized_view.md
index febd2d6..507e56a 100644
--- a/docs/zh-CN/administrator-guide/materialized_view.md
+++ b/docs/zh-CN/administrator-guide/materialized_view.md
@@ -488,7 +488,7 @@ MySQL [test]> desc advertiser_view_record;
2. 如果删除语句的条件列，在物化视图中不存在，则不能进行删除操作。如果一定要删除数据，则需要先将物化视图删除，然后方可删除数据。
3. 单表上过多的物化视图会影响导入的效率：导入数据时，物化视图和 base 表数据是同步更新的，如果一张表的物化视图表超过10张，则有可能导致导入速度很慢。这就像单次导入需要同时导入10张表数据是一样的。
4. 相同列，不同聚合函数，不能同时出现在一张物化视图中，比如：select sum(a), min(a) from table 不支持。
-5. 物化视图针对 Unique Key数据模型，只能改变列顺序，不能起到聚合的作用，所以在Unique Key模型上不能通过创建物化是视图的方式对数据进行粗粒度聚合操作
+5. 物化视图针对 Unique Key数据模型，只能改变列顺序，不能起到聚合的作用，所以在Unique Key模型上不能通过创建物化视图的方式对数据进行粗粒度聚合操作
## 异常错误
1. DATA_QUALITY_ERR: "The data quality does not satisfy, please check your data"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 05/14: [ARM64] Fix some problem when compiling on
ARM64 platform (#6836)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 91fd17df3e1c9a61f36db93c4152f0ca65b44a93
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Sat Oct 16 21:56:49 2021 +0800
[ARM64] Fix some problem when compiling on ARM64 platform (#6836)
1. Refactor the create method of hdfs reader & writer.
libhdfs3 does not support arm64. So we should not support hdfs reader & writer on arm64.
2. And micro for LowerUpperImpl
---
be/src/exec/CMakeLists.txt | 3 +-
be/src/exec/broker_scanner.cpp | 13 ++-----
be/src/exec/hdfs_file_reader.cpp | 3 +-
be/src/exec/hdfs_file_reader.h | 2 +-
be/src/exec/hdfs_reader_writer.cpp | 51 ++++++++++++++++++++++++++
be/src/exec/hdfs_reader_writer.h | 44 ++++++++++++++++++++++
be/src/exec/hdfs_writer.cpp | 2 +-
be/src/exec/hdfs_writer.h | 2 +-
be/src/exec/parquet_scanner.cpp | 13 ++-----
be/src/glibc-compatibility/CMakeLists.txt | 4 +-
be/src/runtime/file_result_writer.cpp | 6 +--
be/src/util/vectorized-tool/lower_upper_impl.h | 4 +-
docs/en/installing/compilation-arm.md | 3 +-
docs/zh-CN/installing/compilation-arm.md | 2 +-
14 files changed, 122 insertions(+), 30 deletions(-)
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index dd8b0e5..5d4b631 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -105,13 +105,14 @@ set(EXEC_FILES
assert_num_rows_node.cpp
s3_reader.cpp
s3_writer.cpp
- hdfs_writer.cpp
+ hdfs_reader_writer.cpp
)
if (ARCH_AMD64)
set(EXEC_FILES
${EXEC_FILES}
hdfs_file_reader.cpp
+ hdfs_writer.cpp
)
endif()
diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp
index 91b2407..a260035 100644
--- a/be/src/exec/broker_scanner.cpp
+++ b/be/src/exec/broker_scanner.cpp
@@ -40,9 +40,7 @@
#include "runtime/tuple.h"
#include "util/utf8_check.h"
-#if defined(__x86_64__)
-#include "exec/hdfs_file_reader.h"
-#endif
+#include "exec/hdfs_reader_writer.h"
namespace doris {
@@ -168,15 +166,12 @@ Status BrokerScanner::open_file_reader() {
break;
}
case TFileType::FILE_HDFS: {
-#if defined(__x86_64__)
- BufferedReader* file_reader =
- new BufferedReader(_profile, new HdfsFileReader(range.hdfs_params, range.path, start_offset));
+ FileReader* hdfs_file_reader;
+ RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params, range.path, start_offset, &hdfs_file_reader));
+ BufferedReader* file_reader = new BufferedReader(_profile, hdfs_file_reader);
RETURN_IF_ERROR(file_reader->open());
_cur_file_reader = file_reader;
break;
-#else
- return Status::InternalError("HdfsFileReader do not support on non x86 platform");
-#endif
}
case TFileType::FILE_BROKER: {
BrokerReader* broker_reader =
diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp
index 6c05840..a6f509f 100644
--- a/be/src/exec/hdfs_file_reader.cpp
+++ b/be/src/exec/hdfs_file_reader.cpp
@@ -22,7 +22,8 @@
#include "common/logging.h"
namespace doris {
-HdfsFileReader::HdfsFileReader(THdfsParams hdfs_params, const std::string& path,
+
+HdfsFileReader::HdfsFileReader(const THdfsParams& hdfs_params, const std::string& path,
int64_t start_offset)
: _hdfs_params(hdfs_params),
_path(path),
diff --git a/be/src/exec/hdfs_file_reader.h b/be/src/exec/hdfs_file_reader.h
index df055da..d4430de 100644
--- a/be/src/exec/hdfs_file_reader.h
+++ b/be/src/exec/hdfs_file_reader.h
@@ -26,7 +26,7 @@ namespace doris {
class HdfsFileReader : public FileReader {
public:
- HdfsFileReader(THdfsParams hdfs_params, const std::string& path, int64_t start_offset);
+ HdfsFileReader(const THdfsParams& hdfs_params, const std::string& path, int64_t start_offset);
virtual ~HdfsFileReader();
virtual Status open() override;
diff --git a/be/src/exec/hdfs_reader_writer.cpp b/be/src/exec/hdfs_reader_writer.cpp
new file mode 100644
index 0000000..86109c3
--- /dev/null
+++ b/be/src/exec/hdfs_reader_writer.cpp
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/hdfs_reader_writer.h"
+
+#if defined(__x86_64__)
+#include "exec/hdfs_file_reader.h"
+#include "exec/hdfs_writer.h"
+#endif
+
+namespace doris {
+
+Status HdfsReaderWriter::create_reader(const THdfsParams& hdfs_params,
+ const std::string& path,
+ int64_t start_offset,
+ FileReader** reader) {
+#if defined(__x86_64__)
+ *reader = new HdfsFileReader(hdfs_params, path, start_offset);
+ return Status::OK();
+#else
+ return Status::InternalError("HdfsFileReader do not support on non x86 platform");
+#endif
+}
+
+Status HdfsReaderWriter::create_writer(std::map<std::string, std::string>& properties,
+ const std::string& path,
+ FileWriter** writer) {
+#if defined(__x86_64__)
+ *writer = new HDFSWriter(properties, path);
+ return Status::OK();
+#else
+ return Status::InternalError("HdfsWriter do not support on non x86 platform");
+#endif
+
+}
+
+} // namespace doris
diff --git a/be/src/exec/hdfs_reader_writer.h b/be/src/exec/hdfs_reader_writer.h
new file mode 100644
index 0000000..996171a
--- /dev/null
+++ b/be/src/exec/hdfs_reader_writer.h
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exec/file_reader.h"
+#include "exec/file_writer.h"
+#include "gen_cpp/PlanNodes_types.h"
+
+namespace doris {
+
+// This class is used to create hdfs readers and writers.
+// Because libhdfs3 does not support the arm64 environment,
+// we use this class to shield the upper layer from the need to deal with the platform environment
+// when creating a raeder or writer.
+//
+// If in the arm64 environment, creating a reader or writer through this class will return an error.
+class HdfsReaderWriter {
+public:
+ static Status create_reader(const THdfsParams& hdfs_params,
+ const std::string& path,
+ int64_t start_offset,
+ FileReader** reader);
+
+ static Status create_writer(std::map<std::string, std::string>& properties,
+ const std::string& path,
+ FileWriter** writer);
+};
+
+} // namespace doris
diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp
index 6c4b343..a00413f 100644
--- a/be/src/exec/hdfs_writer.cpp
+++ b/be/src/exec/hdfs_writer.cpp
@@ -170,4 +170,4 @@ Status HDFSWriter::_parse_properties(std::map<std::string, std::string>& prop) {
return Status::OK();
}
-}// end namespace doris
\ No newline at end of file
+}// end namespace doris
diff --git a/be/src/exec/hdfs_writer.h b/be/src/exec/hdfs_writer.h
index 19e0832..a3f17ec 100644
--- a/be/src/exec/hdfs_writer.h
+++ b/be/src/exec/hdfs_writer.h
@@ -54,4 +54,4 @@ private:
bool _closed = false;
};
-}
\ No newline at end of file
+}
diff --git a/be/src/exec/parquet_scanner.cpp b/be/src/exec/parquet_scanner.cpp
index 86107d9..f574c83 100644
--- a/be/src/exec/parquet_scanner.cpp
+++ b/be/src/exec/parquet_scanner.cpp
@@ -42,9 +42,7 @@
#include "exec/decompressor.h"
#include "exec/parquet_reader.h"
-#if defined(__x86_64__)
- #include "exec/hdfs_file_reader.h"
-#endif
+#include "exec/hdfs_reader_writer.h"
namespace doris {
@@ -131,13 +129,10 @@ Status ParquetScanner::open_next_reader() {
break;
}
case TFileType::FILE_HDFS: {
-#if defined(__x86_64__)
- file_reader.reset(new HdfsFileReader(
- range.hdfs_params, range.path, range.start_offset));
+ FileReader* reader;
+ RETURN_IF_ERROR(HdfsReaderWriter::create_reader(range.hdfs_params, range.path, range.start_offset, &reader));
+ file_reader.reset(reader);
break;
-#else
- return Status::InternalError("HdfsFileReader do not support on non x86 platform");
-#endif
}
case TFileType::FILE_BROKER: {
int64_t file_size = 0;
diff --git a/be/src/glibc-compatibility/CMakeLists.txt b/be/src/glibc-compatibility/CMakeLists.txt
index 19b1caf..40dd6fd 100644
--- a/be/src/glibc-compatibility/CMakeLists.txt
+++ b/be/src/glibc-compatibility/CMakeLists.txt
@@ -32,10 +32,12 @@ if (GLIBC_COMPATIBILITY)
add_headers_and_sources(glibc_compatibility .)
add_headers_and_sources(glibc_compatibility musl)
if (ARCH_ARM)
+ # FastMemcpy not support arm, remove it from glibc_compatibility_sources
+ list (REMOVE_ITEM glibc_compatibility_sources FastMemcpy.c memcpy_wrapper.c)
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s)
set (musl_arch_include_dir musl/aarch64)
elseif (ARCH_AMD64)
- list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s FastMemcpy.c)
+ list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s)
set (musl_arch_include_dir musl/x86_64)
else ()
message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
diff --git a/be/src/runtime/file_result_writer.cpp b/be/src/runtime/file_result_writer.cpp
index 5fd25b3..a71444d 100644
--- a/be/src/runtime/file_result_writer.cpp
+++ b/be/src/runtime/file_result_writer.cpp
@@ -21,7 +21,7 @@
#include "exec/local_file_writer.h"
#include "exec/parquet_writer.h"
#include "exec/s3_writer.h"
-#include "exec/hdfs_writer.h"
+#include "exec/hdfs_reader_writer.h"
#include "exprs/expr.h"
#include "exprs/expr_context.h"
#include "gen_cpp/PaloInternalService_types.h"
@@ -137,9 +137,9 @@ Status FileResultWriter::_create_file_writer(const std::string& file_name) {
new BrokerWriter(_state->exec_env(), _file_opts->broker_addresses,
_file_opts->broker_properties, file_name, 0 /*start offset*/);
} else if (_storage_type == TStorageBackendType::S3) {
- _file_writer = new S3Writer(_file_opts->broker_properties, file_name, 0 /* offset */);
+ _file_writer = new S3Writer(_file_opts->broker_properties, file_name, 0 /* offset */);
} else if (_storage_type == TStorageBackendType::HDFS) {
- _file_writer = new HDFSWriter(const_cast<std::map<std::string, std::string>&>(_file_opts->broker_properties), file_name);
+ RETURN_IF_ERROR(HdfsReaderWriter::create_writer(const_cast<std::map<std::string, std::string>&>(_file_opts->broker_properties), file_name, &_file_writer));
}
RETURN_IF_ERROR(_file_writer->open());
switch (_file_opts->file_format) {
diff --git a/be/src/util/vectorized-tool/lower_upper_impl.h b/be/src/util/vectorized-tool/lower_upper_impl.h
index 11612dd..6d7853f 100644
--- a/be/src/util/vectorized-tool/lower_upper_impl.h
+++ b/be/src/util/vectorized-tool/lower_upper_impl.h
@@ -16,7 +16,9 @@
// under the License.
#pragma once
+#ifdef __SSE2__
#include <emmintrin.h>
+#endif
#include <stdint.h>
#include <iostream>
#include <string>
@@ -58,4 +60,4 @@ public:
*dst = *src;
}
};
-}
\ No newline at end of file
+}
diff --git a/docs/en/installing/compilation-arm.md b/docs/en/installing/compilation-arm.md
index 58a5d40..19276d8 100644
--- a/docs/en/installing/compilation-arm.md
+++ b/docs/en/installing/compilation-arm.md
@@ -220,4 +220,5 @@ Suppose Doris source code is under `/home/doris/doris-src/`.
### 4. Compile Doris source code
-Just execute sh build.sh.
+Execute `sh build.sh`.
+
diff --git a/docs/zh-CN/installing/compilation-arm.md b/docs/zh-CN/installing/compilation-arm.md
index 7a26f17..fbf750f 100644
--- a/docs/zh-CN/installing/compilation-arm.md
+++ b/docs/zh-CN/installing/compilation-arm.md
@@ -220,4 +220,4 @@ make -j && make install
### 4. 编译Doris源码
-执行 sh build.sh 即可。
+执行 `sh build.sh` 即可。
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 06/14: [BUG] Fix printing
ReservationTrackerCounters cause BE crash when mem_limit is reached (#6849)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit ba5a3766867d8f83d4dcee4167156bbb94a88327
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Sat Oct 16 21:57:09 2021 +0800
[BUG] Fix printing ReservationTrackerCounters cause BE crash when mem_limit is reached (#6849)
When the memory usage of BE reaches mem_limit, printing ReservationTrackerCounters through MemTracker
may cause BE crash in high concurrency.
ReservationTrackerCounters is not actually used in the current Doris, and the memory tracker in Doris
will be redesigned in the future.
---
be/src/runtime/mem_tracker.cpp | 23 +++++++++++++----------
1 file changed, 13 insertions(+), 10 deletions(-)
diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp
index ea0a169..ab8a5aa 100644
--- a/be/src/runtime/mem_tracker.cpp
+++ b/be/src/runtime/mem_tracker.cpp
@@ -369,16 +369,19 @@ std::string MemTracker::LogUsage(int max_recursive_depth, const string& prefix,
if (CheckLimitExceeded(MemLimit::HARD)) ss << " memory limit exceeded.";
if (limit_ > 0) ss << " Limit=" << PrettyPrinter::print(limit_, TUnit::BYTES);
- ReservationTrackerCounters* reservation_counters = reservation_counters_.load();
- if (reservation_counters != nullptr) {
- int64_t reservation = reservation_counters->peak_reservation->current_value();
- ss << " Reservation=" << PrettyPrinter::print(reservation, TUnit::BYTES);
- if (reservation_counters->reservation_limit != nullptr) {
- int64_t limit = reservation_counters->reservation_limit->value();
- ss << " ReservationLimit=" << PrettyPrinter::print(limit, TUnit::BYTES);
- }
- ss << " OtherMemory=" << PrettyPrinter::print(curr_consumption - reservation, TUnit::BYTES);
- }
+ // TODO(zxy): ReservationTrackerCounters is not actually used in the current Doris.
+ // Printing here ReservationTrackerCounters may cause BE crash when high concurrency.
+ // The memory tracker in Doris will be redesigned in the future.
+ // ReservationTrackerCounters* reservation_counters = reservation_counters_.load();
+ // if (reservation_counters != nullptr) {
+ // int64_t reservation = reservation_counters->peak_reservation->current_value();
+ // ss << " Reservation=" << PrettyPrinter::print(reservation, TUnit::BYTES);
+ // if (reservation_counters->reservation_limit != nullptr) {
+ // int64_t limit = reservation_counters->reservation_limit->value();
+ // ss << " ReservationLimit=" << PrettyPrinter::print(limit, TUnit::BYTES);
+ // }
+ // ss << " OtherMemory=" << PrettyPrinter::print(curr_consumption - reservation, TUnit::BYTES);
+ // }
ss << " Total=" << PrettyPrinter::print(curr_consumption, TUnit::BYTES);
// Peak consumption is not accurate if the metric is lazily updated (i.e.
// this is a non-root tracker that exists only for reporting purposes).
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 13/14: [Doc] Add documents for MySQL Binlog Load
(#6859)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit ead82c0e27dd09017f93c76454e14d864b6038a7
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Tue Oct 19 10:25:42 2021 +0800
[Doc] Add documents for MySQL Binlog Load (#6859)
* add zh-CN docs
* add en docs and image
* fix
* fix
---
docs/.vuepress/public/images/canal_store.png | Bin 0 -> 51780 bytes
docs/.vuepress/sidebar/en.js | 6 +
docs/.vuepress/sidebar/zh-CN.js | 6 +
.../load-data/binlog-load-manual.md | 492 +++++++++++++++++++++
.../Data Manipulation/CREATE SYNC JOB.md | 165 +++++++
.../Data Manipulation/PAUSE SYNC JOB.md | 48 ++
.../Data Manipulation/RESUME SYNC JOB.md | 46 ++
.../Data Manipulation/SHOW SYNC JOB.md | 49 ++
.../Data Manipulation/STOP SYNC JOB.md | 44 ++
.../load-data/binlog-load-manual.md | 470 ++++++++++++++++++++
.../Data Manipulation/CREATE SYNC JOB.md | 166 +++++++
.../Data Manipulation/PAUSE SYNC JOB.md | 43 ++
.../Data Manipulation/RESUME SYNC JOB.md | 44 ++
.../Data Manipulation/SHOW SYNC JOB.md | 49 ++
.../Data Manipulation/STOP SYNC JOB.md | 44 ++
15 files changed, 1672 insertions(+)
diff --git a/docs/.vuepress/public/images/canal_store.png b/docs/.vuepress/public/images/canal_store.png
new file mode 100644
index 0000000..5167158
Binary files /dev/null and b/docs/.vuepress/public/images/canal_store.png differ
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index b2947b5..fe27f66 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -55,6 +55,7 @@ module.exports = [
children: [
"load-manual",
"batch-delete-manual",
+ "binlog-load-manual",
"broker-load-manual",
"routine-load-manual",
"sequence-column-manual",
@@ -531,6 +532,7 @@ module.exports = [
"CANCEL DELETE",
"CANCEL LABEL",
"CANCEL LOAD",
+ "CREATE SYNC JOB",
"DELETE",
"EXPORT",
"GET LABEL STATE",
@@ -539,8 +541,10 @@ module.exports = [
"MINI LOAD",
"MULTI LOAD",
"PAUSE ROUTINE LOAD",
+ "PAUSE SYNC JOB",
"RESTORE TABLET",
"RESUME ROUTINE LOAD",
+ "RESUME SYNC JOB",
"ROUTINE LOAD",
"SHOW ALTER",
"SHOW BACKUP",
@@ -559,10 +563,12 @@ module.exports = [
"SHOW ROUTINE LOAD TASK",
"SHOW ROUTINE LOAD",
"SHOW SNAPSHOT",
+ "SHOW SYNC JOB",
"SHOW TABLES",
"SHOW TABLET",
"SHOW TRANSACTION",
"STOP ROUTINE LOAD",
+ "STOP SYNC JOB",
"STREAM LOAD",
"alter-routine-load",
"insert",
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index 27c46a8..6256acc 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -54,6 +54,7 @@ module.exports = [
children: [
"load-manual",
"batch-delete-manual",
+ "binlog-load-manual",
"broker-load-manual",
"routine-load-manual",
"sequence-column-manual",
@@ -535,6 +536,7 @@ module.exports = [
children: [
"BROKER LOAD",
"CANCEL LOAD",
+ "CREATE SYNC JOB",
"DELETE",
"EXPORT",
"GROUP BY",
@@ -542,7 +544,9 @@ module.exports = [
"MINI LOAD",
"MULTI LOAD",
"PAUSE ROUTINE LOAD",
+ "PAUSE SYNC JOB",
"RESUME ROUTINE LOAD",
+ "RESUME SYNC JOB",
"ROUTINE LOAD",
"SHOW ALTER",
"SHOW BACKUP",
@@ -561,11 +565,13 @@ module.exports = [
"SHOW ROUTINE LOAD TASK",
"SHOW ROUTINE LOAD",
"SHOW SNAPSHOT",
+ "SHOW SYNC JOB",
"SHOW TABLES",
"SHOW TABLET",
"SHOW TRANSACTION",
"SPARK LOAD",
"STOP ROUTINE LOAD",
+ "STOP SYNC JOB",
"STREAM LOAD",
"alter-routine-load",
"insert",
diff --git a/docs/en/administrator-guide/load-data/binlog-load-manual.md b/docs/en/administrator-guide/load-data/binlog-load-manual.md
new file mode 100644
index 0000000..90db6da
--- /dev/null
+++ b/docs/en/administrator-guide/load-data/binlog-load-manual.md
@@ -0,0 +1,492 @@
+---
+{
+ "title": "Binlog Load",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Binlog Load
+
+The Binlog Load feature enables Doris to incrementally synchronize update operations in MySQL, so as to CDC(Change Data Capture) of data on Mysql.
+
+## Scenarios
+* Support insert / update / delete operations
+* Filter query
+* Temporarily incompatible with DDL statements
+
+## Glossary
+* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access.
+* BE: Backend, the backend node of Doris. Responsible for query execution and data storage.
+* Canal: Alibaba's open source MySQL binlog parsing tool. Support incremental data subscription & consumption.
+* Batch: A batch of data sent by canal to the client with a globally unique self-incrementing ID.
+* SyncJob: A data synchronization job submitted by the user.
+* Receiver: Responsible for subscribing to and receiving data from canal.
+* Consumer: Responsible for distributing the data received by the Receiver to each channel.
+* Channel: The channel that receives the data distributed by Consumer, it creates tasks for sending data, and controls the begining, committing and aborting of transaction in one table.
+* Task: Task created by channel, sends data to Be when executing.
+
+## Principle
+In the design of phase one, Binlog Load needs to rely on canal as an intermediate medium, so that canal can be pretended to be a slave node to get and parse the binlog on the MySQL master node, and then Doris can get the parsed data on the canal. This process mainly involves mysql, canal and Doris. The overall data flow is as follows:
+
+```
++---------------------------------------------+
+| Mysql |
++----------------------+----------------------+
+ | Binlog
++----------------------v----------------------+
+| Canal Server |
++-------------------+-----^-------------------+
+ Get | | Ack
++-------------------|-----|-------------------+
+| FE | | |
+| +-----------------|-----|----------------+ |
+| | Sync Job | | | |
+| | +------------v-----+-----------+ | |
+| | | Canal Client | | |
+| | | +-----------------------+ | | |
+| | | | Receiver | | | |
+| | | +-----------------------+ | | |
+| | | +-----------------------+ | | |
+| | | | Consumer | | | |
+| | | +-----------------------+ | | |
+| | +------------------------------+ | |
+| +----+---------------+--------------+----+ |
+| | | | |
+| +----v-----+ +-----v----+ +-----v----+ |
+| | Channel1 | | Channel2 | | Channel3 | |
+| | [Table1] | | [Table2] | | [Table3] | |
+| +----+-----+ +-----+----+ +-----+----+ |
+| | | | |
+| +--|-------+ +---|------+ +---|------+|
+| +---v------+| +----v-----+| +----v-----+||
+| +----------+|+ +----------+|+ +----------+|+|
+| | Task |+ | Task |+ | Task |+ |
+| +----------+ +----------+ +----------+ |
++----------------------+----------------------+
+ | | |
++----v-----------------v------------------v---+
+| Coordinator |
+| BE |
++----+-----------------+------------------+---+
+ | | |
++----v---+ +---v----+ +----v---+
+| BE | | BE | | BE |
++--------+ +--------+ +--------+
+
+```
+
+As shown in the figure above, the user first submits a SyncJob to the Fe.
+
+Then, Fe will start a Canal Client for each SyncJob to subscribe to and get data from the Canal Server.
+
+The Receiver in the Canal Client will receives data by the GET request. Every time a Batch is received, it will be distributed by the Consumer to different Channels according to the corresponding target table. Once a channel received data distributed by Consumer, it will submit a send task for sending data.
+
+A Send task is a request from Channel to Be, which contains the data of the same Batch distributed to the current channel.
+
+Channel controls the begin, commit and abort of transaction of single table. In a transaction, the consumer may distribute multiple Batches of data to a channel, so multiple send tasks may be generated. These tasks will not actually take effect until the transaction is committed successfully.
+
+When certain conditions are met (for example, a certain period of time was passed, an empty batch is received), the Consumer will block and notify each channel to try commit the transaction.
+
+If and only if all channels are committed successfully, Canal Server will be notified by the ACK request and Canal Client continue to get and consume data.
+
+If there are any Channel fails to commit, it will retrieve data from the location where the last consumption was successful and commit again (the Channel that has successfully commited before will not commmit again to ensure the idempotency of commit).
+
+In the whole cycle of a SyncJob, Canal Client continuously received data from Canal Server and send it to Be through the above process to complete data synchronization.
+
+## Configure MySQL Server
+
+In the master-slave synchronization of MySQL Cluster mode, the binary log file (binlog) records all data changes on the master node. Data synchronization and backup among multiple nodes of the cluster should be carried out through binlog logs, so as to improve the availability of the cluster.
+
+The architecture of master-slave synchronization is usually composed of a master node (responsible for writing) and one or more slave nodes (responsible for reading). All data changes on the master node will be copied to the slave node.
+
+**Note that: Currently, you must use MySQL version 5.7 or above to support Binlog Load**
+
+To enable the binlog of MySQL, you need to edit the my.cnf file and set it like:
+
+```
+[mysqld]
+log-bin = mysql-bin # 开启 binlog
+binlog-format=ROW # 选择 ROW 模式
+```
+
+### Principle Description
+
+On MySQL, the binlog files usually name as mysql-bin.000001, mysql-bin.000002... And MySQL will automatically segment the binlog file when certain conditions are met:
+
+1. MySQL is restarted
+2. The user enters the `flush logs` command
+3. The binlog file size exceeds 1G
+
+To locate the latest consumption location of binlog, the binlog file name and position (offset) must be needed.
+
+For instance, the binlog location of the current consumption so far will be saved on each slave node to prepare for disconnection, reconnection and continued consumption at any time.
+
+```
+--------------------- ---------------------
+| Slave | read | Master |
+| FileName/Position | <<<--------------------------- | Binlog Files |
+--------------------- ---------------------
+```
+
+For the master node, it is only responsible for writing to the binlog. Multiple slave nodes can be connected to a master node at the same time to consume different parts of the binlog log without affecting each other.
+
+Binlog log supports two main formats (in addition to mixed based mode):
+
+* Statement-based format:
+
+ Binlog only records the SQL statements executed on the master node, and the slave node copies them to the local node for re-execution.
+
+* Row-based format:
+
+ Binlog will record the data change information of each row and all columns of the master node, and the slave node will copy and execute the change of each row to the local node.
+
+The first format only writes the executed SQL statements. Although the log volume will be small, it has the following disadvantages:
+
+1. The actual data of each row is not recorded
+2. The UDF, random and time functions executed on the master node will have inconsistent results on the slave node
+3. The execution order of limit statements may be inconsistent
+
+Therefore, we need to choose the second format which parses each row of data from the binlog log.
+
+In the row-based format, binlog will record the timestamp, server ID, offset and other information of each binlog event. For instance, the following transaction with two insert statements:
+
+```
+begin;
+insert into canal_test.test_tbl values (3, 300);
+insert into canal_test.test_tbl values (4, 400);
+commit;
+```
+
+There will be four binlog events, including one begin event, two insert events and one commit event:
+
+```
+SET TIMESTAMP=1538238301/*!*/;
+BEGIN
+/*!*/.
+# at 211935643
+# at 211935698
+#180930 0:25:01 server id 1 end_log_pos 211935698 Table_map: 'canal_test'.'test_tbl' mapped to number 25
+#180930 0:25:01 server id 1 end_log_pos 211935744 Write_rows: table-id 25 flags: STMT_END_F
+...
+'/*!*/;
+### INSERT INTO canal_test.test_tbl
+### SET
+### @1=1
+### @2=100
+# at 211935744
+#180930 0:25:01 server id 1 end_log_pos 211935771 Xid = 2681726641
+...
+'/*!*/;
+### INSERT INTO canal_test.test_tbl
+### SET
+### @1=2
+### @2=200
+# at 211935771
+#180930 0:25:01 server id 1 end_log_pos 211939510 Xid = 2681726641
+COMMIT/*!*/;
+```
+
+As shown above, each insert event contains modified data. During delete/update, an event can also contain multiple rows of data, making the binlog more compact.
+
+### Open GTID mode (Optional)
+
+A global transaction ID (global transaction identifier) identifies a transaction that has been committed on the master node, which is unique and valid in global. After binlog is enabled, the gtid will be written to the binlog file.
+
+To open the gtid mode of MySQL, you need to edit the my.cnf configuration file and set it like:
+
+```
+gtid-mode=on // Open gtid mode
+enforce-gtid-consistency=1 // Enforce consistency between gtid and transaction
+```
+
+In gtid mode, the master server can easily track transactions, recover data and replicas without binlog file name and offset.
+
+In gtid mode, due to the global validity of gtid, the slave node will no longer need to locate the binlog location on the master node by saving the file name and offset, but can be located by the data itself. During SyncJob, the slave node will skip the execution of any gtid transaction already executed before.
+
+Gtid is expressed as a pair of coordinates, `source_ID` identifies the master node, `transaction_ID` indicates the order in which this transaction is executed on the master node (max 2<sup>63</sup>-1).
+
+```
+GTID = source_id:transaction_id
+```
+
+For example, the gtid of the 23rd transaction executed on the same master node is:
+
+```
+3E11FA47-71CA-11E1-9E33-C80AA9429562:23
+```
+
+## Configure Canal Server
+
+Canal is a sub project of Alibaba Otter project. Its main purpose is to provide incremental data subscription and consumption based on MySQL database binlog analysis, which is originally used to solve the scenario of cross machine-room synchronization.
+
+Canal version 1.1.5 and above is recommended. [download link](https://github.com/alibaba/canal/releases)
+
+After downloading, please follow the steps below to complete the deployment.
+
+1. Unzip the canal deployer
+2. Create a new directory under the conf folder and rename it as the root directory of instance. The directory name is the destination mentioned later.
+3. Modify the instance configuration file (you can copy from `conf/example/instance.properties`)
+
+ ```
+ vim conf/{your destination}/instance.properties
+ ```
+ ```
+ ## canal instance serverId
+ canal.instance.mysql.slaveId = 1234
+ ## mysql adress
+ canal.instance.master.address = 127.0.0.1:3306
+ ## mysql username/password
+ canal.instance.dbUsername = canal
+ canal.instance.dbPassword = canal
+ ```
+4. start up canal server
+
+ ```
+ sh bin/startup.sh
+ ```
+
+5. Validation start up successfully
+
+ ```
+ cat logs/{your destination}/{your destination}.log
+ ```
+ ```
+ 2013-02-05 22:50:45.636 [main] INFO c.a.o.c.i.spring.support.PropertyPlaceholderConfigurer - Loading properties file from class path resource [canal.properties]
+ 2013-02-05 22:50:45.641 [main] INFO c.a.o.c.i.spring.support.PropertyPlaceholderConfigurer - Loading properties file from class path resource [xxx/instance.properties]
+ 2013-02-05 22:50:45.803 [main] INFO c.a.otter.canal.instance.spring.CanalInstanceWithSpring - start CannalInstance for 1-xxx
+ 2013-02-05 22:50:45.810 [main] INFO c.a.otter.canal.instance.spring.CanalInstanceWithSpring - start successful....
+ ```
+
+### Principle Description
+
+By faking its own MySQL dump protocol, canal disguises itself as a slave node, get and parses the binlog of the master node.
+
+Multiple instances can be started on the canal server. An instance can be regarded as a slave node. Each instance consists of the following parts:
+
+```
+-------------------------------------------------
+| Server |
+| -------------------------------------------- |
+| | Instance 1 | |
+| | ----------- ----------- ----------- | |
+| | | Parser | | Sink | | Store | | |
+| | ----------- ----------- ----------- | |
+| | ----------------------------------- | |
+| | | MetaManager | | |
+| | ----------------------------------- | |
+| -------------------------------------------- |
+-------------------------------------------------
+```
+
+* Parser: Access the data source, simulate the dump protocol, interact with the master, and analyze the protocol
+* Sink: Linker between parser and store, for data filtering, processing and distribution
+* Store: Data store
+* Meta Manager: Metadata management module
+
+Each instance has its own unique ID in the cluster, that is, server ID.
+
+In the canal server, the instance is identified by a unique string named destination. The canal client needs destination to connect to the corresponding instance.
+
+**Note that: canal client and canal instance should correspond to each other one by one**
+
+Binlog load has forbidded multiple SyncJobs to connect to the same destination.
+
+The data flow direction in instance is binlog -> Parser -> sink -> store.
+
+Instance parses binlog logs through the parser module, and the parsed data is cached in the store. When the user submits a SyncJob to Fe, it will start a Canal Client to subscripe and get the data in the store in the corresponding instance.
+
+The store is actually a ring queue. Users can configure its length and storage space by themselves.
+
+
+
+Store manages the data in the queue through three pointers:
+
+1. Get pointer: the GET pointer points to the last location get by the Canal Client.
+2. Ack pointer: the ACK pointer points to the location of the last successful consumption.
+3. Put pointer: the PUT pointer points to the location where the sink module successfully wrote to the store at last.
+
+```
+canal client asynchronously get data in the store
+
+ get 0 get 1 get 2 put
+ | | | ...... |
+ v v v v
+--------------------------------------------------------------------- store ring queue
+ ^ ^
+ | |
+ ack 0 ack 1
+```
+
+When the Canal Client calls the Get command, the Canal Server will generate data batches and send them to the Canal Client, and move the Get pointer to the right. The Canal Client can get multiple batches until the Get pointer catches up with the Put pointer.
+
+When the consumption of data is successful, the Canal Client will return Ack + Batch ID, notify that the consumption has been successful, and move the Ack pointer to the right. The store will delete the data of this batch from the ring queue, make room to get data from the upstream sink module, and then move the Put pointer to the right.
+
+When the data consumption fails, the client will return a rollback notification of the consumption failure, and the store will reset the Get pointer to the left to the Ack pointer's position, so that the next data get by the Canal Client can start from the Ack pointer again.
+
+Like the slave node in mysql, Canal Server also needs to save the latest consumption location of the client. All metadata in Canal Server (such as gtid and binlog location) is managed by the metamanager. At present, these metadata is persisted in the meta.dat file in the instance's root directory in JSON format by default.
+
+## Basic Operation
+
+### Configure Target Table Properties
+
+User needs to first create the target table which is corresponding to the MySQL side.
+
+Binlog Load can only support unique target tables from now, and the batch delete feature of the target table must be activated.
+
+Example:
+
+```
+-- create target table
+CREATE TABLE `test1` (
+ `a` int(11) NOT NULL COMMENT "",
+ `b` int(11) NOT NULL COMMENT ""
+) ENGINE=OLAP
+UNIQUE KEY(`a`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`a`) BUCKETS 8;
+
+-- enable batch delete
+ALTER TABLE canal_test.test1 ENABLE FEATURE "BATCH_DELETE";
+```
+
+### Create SyncJob
+
+The detailed syntax of creating a SyncJob can be viewd in `help create sync job` command. Here we mainly introduce the precautions when creating a SyncJob.
+
+* job_name
+
+ `job_Name` is the unique identifier of the SyncJob in the current database. With a specified job name, only one SyncJob can be running at the same time.
+
+* channel_desc
+
+ `column_Mapping` mainly refers to the mapping relationship between the columns of the MySQL source table and the Doris target table.
+
+ If it is not specified, the columns of the source table and the target table will consider correspond one by one in order.
+
+ However, we still recommend explicitly specifying the mapping relationship of columns, so that when the schema-change of the target table (such as adding a nullable column), data synchronization can still be carried out.
+
+ Otherwise, when the schema-change occur, because the column mapping relationship is no longer one-to-one, the SyncJob will report an error.
+
+* binlog_desc
+
+ `binlog_desc` defines some necessary information for docking the remote binlog address.
+
+ At present, the only supported docking type is the canal type. In canal type, all configuration items need to be prefixed with the canal prefix.
+
+ 1. canal.server.ip: the address of the canal server
+ 2. canal.server.port: the port of canal server
+ 3. canal.destination: the identifier of the instance
+ 4. canal.batchSize: the maximum batch size get from the canal server for each batch. Default 8192
+ 5. canal.username: the username of instance
+ 6. canal.password: the password of instance
+ 7. canal.debug: when set to true, the details message of each batch and each row will be printed, which may affect the performance.
+
+### Show Job Status
+
+Specific commands and examples for showing job status can be found in `help show sync job;` command.
+
+The parameters in the result set have the following meanings:
+
+* State
+
+ The current stage of the job. The transition between job states is shown in the following figure:
+
+ ```
+ +-------------+
+ create job | PENDING | resume job
+ +-----------+ <-------------+
+ | +-------------+ |
+ +----v-------+ +-------+----+
+ | RUNNING | pause job | PAUSED |
+ | +-----------------------> |
+ +----+-------+ run error +-------+----+
+ | +-------------+ |
+ | | CANCELLED | |
+ +-----------> <-------------+
+ stop job +-------------+ stop job
+ system error
+ ```
+
+ After the SyncJob is submitted, the status is pending.
+
+ After the Fe scheduler starts the canal client, the status becomes running.
+
+ User can control the status of the job by three commands: `stop/pause/resume`. After the operation, the job status is `cancelled/paused/running` respectively.
+
+ There is only one final stage of the job, Cancelled. When the job status changes to Canceled, it cannot be resumed again.
+
+ When an error occurs during SyncJob is running, if the error is unrecoverable, the status will change to cancelled, otherwise it will change to paused.
+
+* Channel
+
+ The mapping relationship between all source tables and target tables of the job.
+
+* Status
+
+ The latest consumption location of the current binlog (if the gtid mode is on, the gtid will be displayed), and the delay time of the Doris side compared with the MySQL side.
+
+* JobConfig
+
+ The remote server information of the docking, such as the address of the Canal Server and the destination of the connected instance.
+
+### Control Operation
+
+Users can control the status of jobs through `stop/pause/resume` commands.
+
+You can use `HELP STOP SYNC JOB;`, `HELP PAUSE SYNC JOB`; And `HELP RESUME SYNC JOB;` commands to view help and examples.
+
+## Related Parameters
+
+### Fe configuration
+
+The following configuration belongs to the system level configuration of SyncJob. The configuration value can be modified in configuration file fe.conf.
+
+* `enable_create_sync_job`
+
+ Turn on the Binlog Load feature. The default value is false. This feature is turned off.
+
+* `sync_commit_interval_second`
+
+ Maximum interval time between commit transactions. If there is still data in the channel that has not been committed after this time, the consumer will notify the channel to commit the transaction.
+
+* `max_sync_task_threads_num`
+
+ The maximum number of threads in the SyncJobs' thread pool. There is only one thread pool in the whole Fe for synchronization, which is used to process the tasks created by all SyncJobs in the Fe.
+
+## FAQ
+
+1. Will modifying the table structure affect data synchronization?
+
+ Yes. The SyncJob cannot prohibit `alter table` operation.
+When the table's schema changes, if the column mapping cannot match, the job may be suspended incorrectly. It is recommended to reduce such problems by explicitly specifying the column mapping relationship in the data synchronization job, or by adding nullable columns or columns with default values.
+
+2. Will the SyncJob continue to run after the database is deleted?
+
+ No. In this case, the SyncJob will be checked by the Fe's scheduler thread and be stopped.
+
+3. Can multiple SyncJobs be configured with the same `IP:Port + destination`?
+
+ No. When creating a SyncJob, FE will check whether the `IP:Port + destination` is duplicate with the existing job to prevent multiple jobs from connecting to the same instance.
+
+4. Why is the precision of floating-point type different between MySQL and Doris during data synchronization?
+
+ The precision of Doris floating-point type is different from that of MySQL. You can choose to use decimal type instead.
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md b/docs/en/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md
new file mode 100644
index 0000000..9a46486
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md
@@ -0,0 +1,165 @@
+---
+{
+ "title": "CREATE SYNC JOB",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# CREATE SYNC JOB
+
+## description
+
+The sync job feature supports to submit a resident SyncJob, and CDC (change data capture) the user's update operation in MySQL database by reading the binlog log from the specified remote address.
+
+At present, data synchronization only supports docking with the canal, getting the parsed binlog from the canal server and loading it into Doris.
+
+You can view the SyncJob's status by command 'SHOW SYNC JOB'.
+
+Syntax:
+
+```
+CREATE SYNC [db.]job_name
+ (
+ channel_desc,
+ channel_desc
+ ...
+ )
+binlog_desc
+```
+
+1. `job_name`
+
+ job_Name is the unique identifier of the SyncJob in the current database. With a specified job name, only one SyncJob can be running at the same time.
+
+2. `channel_desc`
+
+ The data channel under the job is used to describe the mapping relationship between the MySQL source table and the Doris target table.
+
+ Syntax:
+
+ ```
+ FROM mysql_db.src_tbl INTO des_tbl
+ [partitions]
+ [columns_mapping]
+ ```
+
+ 1. `mysql_db.src_tbl`
+
+ Specify the database and source table on the MySQL side.
+
+ 2. `des_tbl`
+
+ Specify the target table on the Doris side. Only the unique table is supported, and the batch delete feature of the table needs to be enabled.
+
+ 3. `partitions`
+
+ Specify which partitions to be load into in target table. If it is not specified, it will be automatically loaded into the corresponding partition.
+
+ Example:
+
+ ```
+ PARTITION(p1, p2, p3)
+ ```
+
+ 4. `column_mapping`
+
+ Specify the mapping relationship between the columns of the MySQL source table and the Doris target table. If not specified, Fe will default that the columns of the source table and the target table correspond one by one in order.
+
+ Columns are not supported in the 'col_name = expr' form.
+
+ Example:
+
+ ```
+ Suppose the columns of target table are (K1, K2, V1),
+
+ Change the order of columns K1 and K2
+ COLUMNS(k2, k1, v1)
+
+ Ignore the fourth column of the source data
+ COLUMNS(k2, k1, v1, dummy_column)
+ ```
+
+3. `binlog_desc`
+
+ It is used to describe remote data sources. Currently, only canal is supported.
+
+ Syntax:
+
+ ```
+ FROM BINLOG
+ (
+ "key1" = "value1",
+ "key2" = "value2"
+ )
+ ```
+
+ 1. The attribute related to the canal is prefixed with `canal.`
+
+ 1. canal.server.ip: the address of the canal server
+ 2. canal.server.port: the port of canal server
+ 3. canal.destination: Identifier of instance
+ 4. canal.batchSize: the maximum batch size. The default is 8192
+ 5. canal.username: the username of instance
+ 6. canal.password: password of instance
+ 7. canal.debug: optional. When set to true, the details of each batch and each row will be printed.
+
+## example
+
+1. create a sync job named `job1` for target table `test_tbl` in `test_db`, connects to the local canal server, and corresponds to the MySQL source table `mysql_db1.tbl1`
+
+ CREATE SYNC `test_db`.`job1`
+ (
+ FROM `mysql_db1`.`tbl1` INTO `test_tbl `
+ )
+ FROM BINLOG
+ (
+ "type" = "canal",
+ "canal.server.ip" = "127.0.0.1",
+ "canal.server.port" = "11111",
+ "canal.destination" = "example",
+ "canal.username" = "",
+ "canal.password" = ""
+ );
+
+2. create a sync job named `job1` for multiple target tables in `test_db`, correspond to multiple MySQL source tables one by one, and explicitly specify column mapping.
+
+ CREATE SYNC `test_db`.`job1`
+ (
+ FROM `mysql_db`.`t1` INTO `test1` COLUMNS(k1, k2, v1) PARTITIONS (p1, p2),
+ FROM `mysql_db`.`t2` INTO `test2` COLUMNS(k3, k4, v2) PARTITION p1
+ )
+ FROM BINLOG
+ (
+ "type" = "canal",
+ "canal.server.ip" = "xx.xxx.xxx.xx",
+ "canal.server.port" = "12111",
+ "canal.destination" = "example",
+ "canal.username" = "username",
+ "canal.password" = "password"
+ );
+
+## keyword
+
+ CREATE,SYNC,JOB,BINLOG
+
+
+
diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/PAUSE SYNC JOB.md b/docs/en/sql-reference/sql-statements/Data Manipulation/PAUSE SYNC JOB.md
new file mode 100644
index 0000000..b685270
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Data Manipulation/PAUSE SYNC JOB.md
@@ -0,0 +1,48 @@
+---
+{
+ "title": "PAUSE SYNC JOB",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# PAUSE SYNC JOB
+
+## description
+
+Pauses a running SyncJob in the database.
+
+The paused job will stop synchronizing and keep the latest consumption location until it is resumed by the user.
+
+Syntax:
+
+ PAUSE SYNC JOB [db.]job_name
+
+## example
+
+1. Pause the SyncJob named `job_name`.
+
+ PAUSE SYNC JOB `job_name`;
+
+## keyword
+ PAUSE,SYNC,JOB,BINLOG
+
+
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/RESUME SYNC JOB.md b/docs/en/sql-reference/sql-statements/Data Manipulation/RESUME SYNC JOB.md
new file mode 100644
index 0000000..ef5f7f8
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Data Manipulation/RESUME SYNC JOB.md
@@ -0,0 +1,46 @@
+---
+{
+ "title": "RESUME SYNC JOB",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# RESUME SYNC JOB
+
+## description
+
+Rusumes a paused SyncJob in the database.
+
+The job will continue to synchronize data from the latest location before the last pause.
+
+Syntax:
+
+ RESUME SYNC JOB [db.]job_name
+
+## example
+
+1. Resume the SyncJob named `job_name`
+
+ RESUME SYNC JOB `job_name`;
+
+## keyword
+ RESUME,SYNC,JOB,BINLOG
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW SYNC JOB.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW SYNC JOB.md
new file mode 100644
index 0000000..f496daf
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW SYNC JOB.md
@@ -0,0 +1,49 @@
+---
+{
+ "title": "SHOW SYNC JOB",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# SHOW SYNC JOB
+
+## description
+
+This command is used to display the resident data synchronization job status in all databases.
+
+Syntax:
+
+ SHOW SYNC JOB [FROM db_name]
+
+## example
+
+1. Show the status of all SyncJob in the current database.
+
+ SHOW SYNC JOB;
+
+2. Show status of all SyncJob under databases `test_db`.
+
+ SHOW SYNC JOB FROM `test_db`;
+
+## keyword
+
+ SHOW,SYNC,JOB,BINLOG
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/STOP SYNC JOB.md b/docs/en/sql-reference/sql-statements/Data Manipulation/STOP SYNC JOB.md
new file mode 100644
index 0000000..b9fc262
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Data Manipulation/STOP SYNC JOB.md
@@ -0,0 +1,44 @@
+---
+{
+ "title": "STOP SYNC JOB",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# STOP SYNC JOB
+
+## description
+
+Stops a uncancelled SyncJob in the database.
+
+Syntax:
+
+ STOP SYNC JOB [db.]job_name
+
+## example
+
+1. Stop the SyncJob named `job_name`
+
+ STOP SYNC JOB `job_name`;
+
+## keyword
+ STOP,SYNC,JOB,BINLOG
diff --git a/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md b/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md
new file mode 100644
index 0000000..3ed946b
--- /dev/null
+++ b/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md
@@ -0,0 +1,470 @@
+---
+{
+ "title": "Binlog Load",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+# Binlog Load
+Binlog Load提供了一种使Doris增量同步用户在Mysql数据库的对数据更新操作的CDC(Change Data Capture)功能。
+
+## 适用场景
+
+* INSERT/UPDATE/DELETE支持
+* 过滤Query
+* 暂不兼容DDL语句
+
+## 名词解释
+1. Frontend（FE）：Doris 系统的元数据和调度节点。在导入流程中主要负责导入 plan 生成和导入任务的调度工作。
+2. Backend（BE）：Doris 系统的计算和存储节点。在导入流程中主要负责数据的 ETL 和存储。
+3. Canal：阿里巴巴开源的Mysql Binlog日志解析工具。提供增量数据订阅&消费等功能。
+4. Batch：Canal发送到客户端的一批数据，具有全局唯一自增的ID。
+5. SyncJob：用户提交的一个数据同步作业。
+6. Receiver: 负责订阅并接收Canal的数据。
+7. Consumer: 负责分发Receiver接收的数据到各个Channel。
+8. Channel: 接收Consumer分发的数据的渠道，创建发送数据的子任务，控制单个表事务的开启、提交、终止。
+9. Task：Channel向Be发送数据的子任务。
+
+## 基本原理
+在第一期的设计中，Binlog Load需要依赖canal作为中间媒介，让canal伪造成一个从节点去获取Mysql主节点上的Binlog并解析，再由Doris去获取Canal上解析好的数据，主要涉及Mysql端、Canal端以及Doris端，总体数据流向如下：
+
+```
++---------------------------------------------+
+| Mysql |
++----------------------+----------------------+
+ | Binlog
++----------------------v----------------------+
+| Canal Server |
++-------------------+-----^-------------------+
+ Get | | Ack
++-------------------|-----|-------------------+
+| FE | | |
+| +-----------------|-----|----------------+ |
+| | Sync Job | | | |
+| | +------------v-----+-----------+ | |
+| | | Canal Client | | |
+| | | +-----------------------+ | | |
+| | | | Receiver | | | |
+| | | +-----------------------+ | | |
+| | | +-----------------------+ | | |
+| | | | Consumer | | | |
+| | | +-----------------------+ | | |
+| | +------------------------------+ | |
+| +----+---------------+--------------+----+ |
+| | | | |
+| +----v-----+ +-----v----+ +-----v----+ |
+| | Channel1 | | Channel2 | | Channel3 | |
+| | [Table1] | | [Table2] | | [Table3] | |
+| +----+-----+ +-----+----+ +-----+----+ |
+| | | | |
+| +--|-------+ +---|------+ +---|------+|
+| +---v------+| +----v-----+| +----v-----+||
+| +----------+|+ +----------+|+ +----------+|+|
+| | Task |+ | Task |+ | Task |+ |
+| +----------+ +----------+ +----------+ |
++----------------------+----------------------+
+ | | |
++----v-----------------v------------------v---+
+| Coordinator |
+| BE |
++----+-----------------+------------------+---+
+ | | |
++----v---+ +---v----+ +----v---+
+| BE | | BE | | BE |
++--------+ +--------+ +--------+
+
+```
+
+如上图，用户向FE提交一个数据同步作业。
+
+FE会为每个数据同步作业启动一个canal client，来向canal server端订阅并获取数据。
+
+client中的receiver将负责通过Get命令接收数据，每获取到一个数据batch，都会由consumer根据对应表分发到不同的channel，每个channel都会为此数据batch产生一个发送数据的子任务Task。
+
+在FE上，一个Task是channel向BE发送数据的子任务，里面包含分发到当前channel的同一个batch的数据。
+
+channel控制着单个表事务的开始、提交、终止。一个事务周期内，一般会从consumer获取到多个batch的数据，因此会产生多个向BE发送数据的子任务Task，在提交事务成功前，这些Task不会实际生效。
+
+满足一定条件时（比如超过一定时间、获取到了空的batch），consumer将会阻塞并通知各个channel提交事务。
+
+当且仅当所有channel都提交成功，才会通过Ack命令通知canal并继续获取并消费数据。
+
+如果有任意channel提交失败，将会重新从上一次消费成功的位置获取数据并再次提交（已提交成功的channel不会再次提交以保证幂等性）。
+
+整个数据同步作业中，FE通过以上流程不断的从canal获取数据并提交到BE，来完成数据同步。
+
+## 配置Mysql端
+
+在Mysql Cluster模式的主从同步中，二进制日志文件(Binlog)记录了主节点上的所有数据变化，数据在Cluster的多个节点间同步、备份都要通过Binlog日志进行，从而提高集群的可用性。架构通常由一个主节点(负责写)和一个或多个从节点(负责读)构成，所有在主节点上发生的数据变更将会复制给从节点。
+
+**注意：目前必须要使用Mysql 5.7及以上的版本才能支持Binlog Load功能。**
+
+要打开mysql的二进制binlog日志功能，则需要编辑my.cnf配置文件设置一下。
+
+```
+[mysqld]
+log-bin = mysql-bin # 开启 binlog
+binlog-format=ROW # 选择 ROW 模式
+```
+
+### Mysql端说明
+
+在Mysql上，Binlog命名格式为mysql-bin.000001、mysql-bin.000002... ，满足一定条件时mysql会去自动切分Binlog日志：
+
+1. mysql重启了
+2. 客户端输入命令flush logs
+3. binlog文件大小超过1G
+
+要定位Binlog的最新的消费位置，可以通过binlog文件名和position(偏移量)。
+
+例如，各个从节点上会保存当前消费到的binlog位置，方便随时断开连接、重新连接和继续消费。
+
+```
+--------------------- ---------------------
+| Slave | read | Master |
+| FileName/Position | <<<--------------------------- | Binlog Files |
+--------------------- ---------------------
+```
+
+对于主节点来说，它只负责写入Binlog，多个从节点可以同时连接到一台主节点上，消费Binlog日志的不同部分，互相之间不会影响。
+
+Binlog日志支持两种主要格式(此外还有混合模式mixed-based):
+
+```
+statement-based格式: Binlog只保存主节点上执行的sql语句，从节点将其复制到本地重新执行
+row-based格式: Binlog会记录主节点的每一行所有列的数据的变更信息，从节点会复制并执行每一行的变更到本地
+```
+
+第一种格式只写入了执行的sql语句，虽然日志量会很少，但是有下列缺点
+
+ 1. 没有保存每一行实际的数据
+ 2. 在主节点上执行的UDF、随机、时间函数会在从节点上结果不一致
+ 3. limit语句执行顺序可能不一致
+
+因此我们需要选择第二种格式，才能从Binlog日志中解析出每一行数据。
+
+在row-based格式下，Binlog会记录每一条binlog event的时间戳，server id，偏移量等信息，如下面一条带有两条insert语句的事务:
+
+```
+begin;
+insert into canal_test.test_tbl values (3, 300);
+insert into canal_test.test_tbl values (4, 400);
+commit;
+```
+
+对应将会有四条binlog event，其中一条begin event，两条insert event，一条commit event：
+
+```
+SET TIMESTAMP=1538238301/*!*/;
+BEGIN
+/*!*/.
+# at 211935643
+# at 211935698
+#180930 0:25:01 server id 1 end_log_pos 211935698 Table_map: 'canal_test'.'test_tbl' mapped to number 25
+#180930 0:25:01 server id 1 end_log_pos 211935744 Write_rows: table-id 25 flags: STMT_END_F
+...
+'/*!*/;
+### INSERT INTO canal_test.test_tbl
+### SET
+### @1=1
+### @2=100
+# at 211935744
+#180930 0:25:01 server id 1 end_log_pos 211935771 Xid = 2681726641
+...
+'/*!*/;
+### INSERT INTO canal_test.test_tbl
+### SET
+### @1=2
+### @2=200
+# at 211935771
+#180930 0:25:01 server id 1 end_log_pos 211939510 Xid = 2681726641
+COMMIT/*!*/;
+```
+
+如上图所示，每条Insert event中包含了修改的数据。在进行Delete/Update操作时，一条event还能包含多行数据，使得Binlog日志更加的紧密。
+
+
+
+### 开启GTID模式 [可选]
+一个全局事务Id(global transaction identifier)标识出了一个曾在主节点上提交过的事务，在全局都是唯一有效的。开启了Binlog后，GTID会被写入到Binlog文件中，与事务一一对应。
+
+要打开mysql的GTID模式，则需要编辑my.cnf配置文件设置一下
+
+```
+gtid-mode=on // 开启gtid模式
+enforce-gtid-consistency=1 // 强制gtid和事务的一致性
+```
+
+在GTID模式下，主服务器可以不需要Binlog的文件名和偏移量，就能很方便的追踪事务、恢复数据、复制副本。
+
+在GTID模式下，由于GTID的全局有效性，从节点将不再需要通过保存文件名和偏移量来定位主节点上的Binlog位置，而通过数据本身就可以定位了。在进行数据同步中，从节点会跳过执行任意被识别为已执行的GTID事务。
+
+GTID的表现形式为一对坐标, `source_id`标识出主节点，`transaction_id`表示此事务在主节点上执行的顺序(最大2<sup>63</sup>-1)。
+
+```
+GTID = source_id:transaction_id
+```
+
+例如，在同一主节点上执行的第23个事务的gtid为
+
+```
+3E11FA47-71CA-11E1-9E33-C80AA9429562:23
+```
+
+## 配置Canal端
+canal是属于阿里巴巴otter项目下的一个子项目，主要用途是基于 MySQL 数据库增量日志解析，提供增量数据订阅和消费，用于解决跨机房同步的业务场景，建议使用canal 1.1.5及以上版本，[下载地址](https://github.com/alibaba/canal/releases)，下载完成后，请按以下步骤完成部署。
+
+1. 解压canal deployer
+2. 在conf文件夹下新建目录并重命名，作为instance的根目录，目录名即后文提到的destination
+3. 修改instance配置文件（可拷贝conf/example/instance.properties）
+
+ ```
+ vim conf/{your destination}/instance.properties
+ ```
+ ```
+ ## canal instance serverId
+ canal.instance.mysql.slaveId = 1234
+ ## mysql adress
+ canal.instance.master.address = 127.0.0.1:3306
+ ## mysql username/password
+ canal.instance.dbUsername = canal
+ canal.instance.dbPassword = canal
+ ```
+
+4. 启动
+
+ ```
+ sh bin/startup.sh
+ ```
+
+5. 验证启动成功
+
+ ```
+ cat logs/{your destination}/{your destination}.log
+ ```
+ ```
+ 2013-02-05 22:50:45.636 [main] INFO c.a.o.c.i.spring.support.PropertyPlaceholderConfigurer - Loading properties file from class path resource [canal.properties]
+ 2013-02-05 22:50:45.641 [main] INFO c.a.o.c.i.spring.support.PropertyPlaceholderConfigurer - Loading properties file from class path resource [xxx/instance.properties]
+ 2013-02-05 22:50:45.803 [main] INFO c.a.otter.canal.instance.spring.CanalInstanceWithSpring - start CannalInstance for 1-xxx
+ 2013-02-05 22:50:45.810 [main] INFO c.a.otter.canal.instance.spring.CanalInstanceWithSpring - start successful....
+ ```
+
+### canal端说明
+
+canal通过伪造自己的mysql dump协议，去伪装成一个从节点，获取主节点的Binlog日志并解析。
+
+canal server上可启动多个instance，一个instance可看作一个从节点，每个instance由下面几个部分组成：
+
+```
+-------------------------------------------------
+| Server |
+| -------------------------------------------- |
+| | Instance 1 | |
+| | ----------- ----------- ----------- | |
+| | | Parser | | Sink | | Store | | |
+| | ----------- ----------- ----------- | |
+| | ----------------------------------- | |
+| | | MetaManager | | |
+| | ----------------------------------- | |
+| -------------------------------------------- |
+-------------------------------------------------
+```
+
+* parser：数据源接入，模拟slave协议和master进行交互，协议解析
+* sink：parser和store链接器，进行数据过滤，加工，分发的工作
+* store：数据存储
+* meta manager：元数据管理模块
+
+每个instance都有自己在cluster内的唯一标识，即server Id。
+
+在canal server内，instance用字符串表示，此唯一字符串被记为destination，canal client需要通过destination连接到对应的instance。
+
+**注意：canal client和canal instance是一一对应的**，Binlog Load已限制多个数据同步作业不能连接到同一个destination。
+
+数据在instance内的流向是binlog -> parser -> sink -> store。
+
+instance通过parser模块解析binlog日志，解析出来的数据缓存在store里面，当用户向FE提交一个数据同步作业时，会启动一个canal client订阅并获取对应instance中的store内的数据。
+
+store实际上是一个环形的队列，用户可以自行配置它的长度和存储空间。
+
+
+
+store通过三个指针去管理队列内的数据：
+
+1. get指针：get指针代表客户端最后获取到的位置。
+2. ack指针：ack指针记录着最后消费成功的位置。
+3. put指针：代表sink模块最后写入store成功的位置。
+
+```
+canal client异步获取store中数据
+
+ get 0 get 1 get 2 put
+ | | | ...... |
+ v v v v
+--------------------------------------------------------------------- store环形队列
+ ^ ^
+ | |
+ ack 0 ack 1
+```
+
+canal client调用get命令时，canal server会产生数据batch发送给client，并右移get指针，client可以获取多个batch，直到get指针赶上put指针为止。
+
+当消费数据成功时，client会返回ack + batch Id通知已消费成功了，并右移ack指针，store会从队列中删除此batch的数据，腾出空间来从上游sink模块获取数据，并右移put指针。
+
+当数据消费失败时，client会返回rollback通知消费失败，store会将get指针重置左移到ack指针位置，使下一次client获取的数据能再次从ack指针处开始。
+
+和Mysql中的从节点一样，canal也需要去保存client最新消费到的位置。canal中所有元数据(如GTID、Binlog位置)都是由MetaManager去管理的，目前元数据默认以json格式持久化在instance根目录下的meta.dat文件内。
+
+## 基本操作
+
+### 配置目标表属性
+
+用户需要先在Doris端创建好与Mysql端对应的目标表
+
+Binlog Load只能支持Unique类型的目标表，且必须激活目标表的Batch Delete功能。
+
+开启Batch Delete的方法可以参考`help alter table`中的批量删除功能。
+
+示例：
+
+```
+-- create target table
+CREATE TABLE `test1` (
+ `a` int(11) NOT NULL COMMENT "",
+ `b` int(11) NOT NULL COMMENT ""
+) ENGINE=OLAP
+UNIQUE KEY(`a`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`a`) BUCKETS 8;
+
+-- enable batch delete
+ALTER TABLE canal_test.test1 ENABLE FEATURE "BATCH_DELETE";
+```
+
+### 创建同步作业
+创建数据同步作业的的详细语法可以连接到 Doris 后，执行 HELP CREATE SYNC JOB; 查看语法帮助。这里主要详细介绍，创建作业时的注意事项。
+
+* job_name
+
+ `job_name`是数据同步作业在当前数据库内的唯一标识，相同`job_name`的作业只能有一个在运行。
+
+* channel_desc
+
+ `channel_desc `用来定义任务下的数据通道，可表示mysql源表到doris目标表的映射关系。在设置此项时，如果存在多个映射关系，必须满足mysql源表应该与doris目标表是一一对应关系，其他的任何映射关系（如一对多关系），检查语法时都被视为不合法。
+
+* column_mapping
+
+ `column_mapping`主要指mysql源表和doris目标表的列之间的映射关系，如果不指定，FE会默认源表和目标表的列按顺序一一对应。但是我们依然建议显式的指定列的映射关系，这样当目标表的结构发生变化（比如增加一个 nullable 的列），数据同步作业依然可以进行。否则，当发生上述变动后，因为列映射关系不再一一对应，导入将报错。
+
+* binlog_desc
+
+ `binlog_desc`中的属性定义了对接远端Binlog地址的一些必要信息，目前可支持的对接类型只有canal方式，所有的配置项前都需要加上canal前缀。
+
+ 1. `canal.server.ip`: canal server的地址
+ 2. `canal.server.port`: canal server的端口
+ 3. `canal.destination`: 前文提到的instance的字符串标识
+ 4. `canal.batchSize`: 每批从canal server处获取的batch大小的最大值，默认8192
+ 5. `canal.username`: instance的用户名
+ 6. `canal.password`: instance的密码
+ 7. `canal.debug`: 设置为true时，会将batch和每一行数据的详细信息都打印出来，会影响性能。
+
+### 查看作业状态
+
+查看作业状态的具体命令和示例可以通过 `HELP SHOW SYNC JOB;` 命令查看。
+
+返回结果集的参数意义如下：
+
+* State
+
+ 作业当前所处的阶段。作业状态之间的转换如下图所示：
+
+ ```
+ +-------------+
+ create job | PENDING | resume job
+ +-----------+ <-------------+
+ | +-------------+ |
+ +----v-------+ +-------+----+
+ | RUNNING | pause job | PAUSED |
+ | +-----------------------> |
+ +----+-------+ run error +-------+----+
+ | +-------------+ |
+ | | CANCELLED | |
+ +-----------> <-------------+
+ stop job +-------------+ stop job
+ system error
+ ```
+ 作业提交之后状态为PENDING，由FE调度执行启动canal client后状态变成RUNNING，用户可以通过 STOP/PAUSE/RESUME 三个命令来控制作业的停止，暂停和恢复，操作后作业状态分别为CANCELLED/PAUSED/RUNNING。
+
+ 作业的最终阶段只有一个CANCELLED，当作业状态变为CANCELLED后，将无法再次恢复。当作业发生了错误时，若错误是不可恢复的，状态会变成CANCELLED，否则会变成PAUSED。
+
+* Channel
+
+ 作业所有源表到目标表的映射关系。
+
+* Status
+
+ 当前binlog的消费位置(若设置了GTID模式，会显示GTID)，以及doris端执行时间相比mysql端的延迟时间。
+
+* JobConfig
+
+ 对接的远端服务器信息，如canal server的地址与连接instance的destination
+
+### 控制作业
+用户可以通过 STOP/PAUSE/RESUME 三个命令来控制作业的停止，暂停和恢复。可以通过`HELP STOP SYNC JOB`; `HELP PAUSE SYNC JOB`; 以及 `HELP RESUME SYNC JOB`; 三个命令查看帮助和示例。
+
+## 相关参数
+
+### FE配置
+
+下面配置属于数据同步作业的系统级别配置，主要通过修改 fe.conf 来调整配置值。
+
+* `enable_create_sync_job`
+
+ 开启数据同步作业功能。默认为 false，关闭此功能。
+
+* `sync_commit_interval_second`
+
+ 提交事务的最大时间间隔。若超过了这个时间channel中还有数据没有提交，consumer会通知channel提交事务。
+
+* `max_sync_task_threads_num`
+
+ 数据同步作业线程池中的最大线程数量。此线程池整个FE中只有一个，用于处理FE中所有数据同步作业向BE发送数据的任务task，线程池的实现在`SyncTaskPool`类。
+
+## 常见问题
+
+1. 修改表结构是否会影响数据同步作业？
+
+ 会影响。数据同步作业并不能禁止`alter table`的操作，当表结构发生了变化，如果列的映射无法匹配，可能导致作业发生错误暂停，建议通过在数据同步作业中显式指定列映射关系，或者通过增加 Nullable 列或带 Default 值的列来减少这类问题。
+
+2. 删除了数据库后数据同步作业还会继续运行吗？
+
+ 不会。删除数据库后的几秒日志中可能会出现找不到元数据的错误，之后该数据同步作业会被FE的定时调度检查时停止。
+
+3. 多个数据同步作业可以配置相同的`ip:port + destination`吗？
+
+ 不能。创建数据同步作业时会检查`ip:port + destination`与已存在的作业是否重复，防止出现多个作业连接到同一个instance的情况。
+
+4. 为什么数据同步时浮点类型的数据精度在Mysql端和Doris端不一样？
+
+ Doris本身浮点类型的精度与Mysql不一样。可以选择用Decimal类型代替。
+
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md
new file mode 100644
index 0000000..6705173
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/CREATE SYNC JOB.md
@@ -0,0 +1,166 @@
+---
+{
+ "title": "CREATE SYNC JOB",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# CREATE SYNC JOB
+
+## description
+
+数据同步(Sync Job)功能，支持用户提交一个常驻的数据同步作业，通过从指定的远端地址读取Binlog日志，增量同步用户在Mysql数据库的对数据更新操作的CDC(Change Data Capture)功能。
+
+目前数据同步作业只支持对接Canal，从Canal Server上获取解析好的Binlog数据，导入到Doris内。
+
+用户可通过 `SHOW SYNC JOB` 查看数据同步作业状态。
+
+语法：
+
+```
+CREATE SYNC [db.]job_name
+ (
+ channel_desc,
+ channel_desc
+ ...
+ )
+binlog_desc
+```
+
+1. `job_name`
+
+ 同步作业名称，是作业在当前数据库内的唯一标识，相同`job_name`的作业只能有一个在运行。
+
+2. `channel_desc`
+
+ 作业下的数据通道，用来描述mysql源表到doris目标表的映射关系。
+
+ 语法：
+
+ ```
+ FROM mysql_db.src_tbl INTO des_tbl
+ [partitions]
+ [columns_mapping]
+ ```
+
+ 1. `mysql_db.src_tbl`
+
+ 指定mysql端的数据库和源表。
+
+ 2. `des_tbl`
+
+ 指定doris端的目标表，只支持Unique表，且需开启表的batch delete功能(开启方法请看help alter table的'批量删除功能')。
+
+ 3. `partitions`
+
+ 指定导入目的表的哪些 partition 中。如果不指定，则会自动导入到对应的 partition 中。
+
+ 示例：
+
+ ```
+ PARTITION(p1, p2, p3)
+ ```
+
+ 4. `column_mapping`
+
+ 指定mysql源表和doris目标表的列之间的映射关系。如果不指定，FE会默认源表和目标表的列按顺序一一对应。
+
+ 不支持 col_name = expr 的形式表示列。
+
+ 示例：
+
+ ```
+ 假设目标表列为(k1, k2, v1)，
+
+ 改变列k1和k2的顺序
+ COLUMNS(k2, k1, v1)
+
+ 忽略源数据的第四列
+ COLUMNS(k2, k1, v1, dummy_column)
+ ```
+
+3. `binlog_desc`
+
+ 用来描述远端数据源，目前仅支持canal一种。
+
+ 语法：
+
+ ```
+ FROM BINLOG
+ (
+ "key1" = "value1",
+ "key2" = "value2"
+ )
+ ```
+
+ 1. Canal 数据源对应的属性，以`canal.`为前缀
+
+ 1. canal.server.ip: canal server的地址
+ 2. canal.server.port: canal server的端口
+ 3. canal.destination: instance的标识
+ 4. canal.batchSize: 获取的batch大小的最大值，默认8192
+ 5. canal.username: instance的用户名
+ 6. canal.password: instance的密码
+ 7. canal.debug: 可选，设置为true时，会将batch和每一行数据的详细信息都打印出来
+
+## example
+
+1. 简单为 `test_db` 的 `test_tbl` 创建一个名为 `job1` 的数据同步作业，连接本地的Canal服务器，对应Mysql源表 `mysql_db1.tbl1`。
+
+ CREATE SYNC `test_db`.`job1`
+ (
+ FROM `mysql_db1`.`tbl1` INTO `test_tbl `
+ )
+ FROM BINLOG
+ (
+ "type" = "canal",
+ "canal.server.ip" = "127.0.0.1",
+ "canal.server.port" = "11111",
+ "canal.destination" = "example",
+ "canal.username" = "",
+ "canal.password" = ""
+ );
+
+2. 为 `test_db` 的多张表创建一个名为 `job1` 的数据同步作业，一一对应多张Mysql源表，并显式的指定列映射。
+
+ CREATE SYNC `test_db`.`job1`
+ (
+ FROM `mysql_db`.`t1` INTO `test1` COLUMNS(k1, k2, v1) PARTITIONS (p1, p2),
+ FROM `mysql_db`.`t2` INTO `test2` COLUMNS(k3, k4, v2) PARTITION p1
+ )
+ FROM BINLOG
+ (
+ "type" = "canal",
+ "canal.server.ip" = "xx.xxx.xxx.xx",
+ "canal.server.port" = "12111",
+ "canal.destination" = "example",
+ "canal.username" = "username",
+ "canal.password" = "password"
+ );
+
+## keyword
+
+ CREATE,SYNC,JOB,BINLOG
+
+
+
+
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/PAUSE SYNC JOB.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/PAUSE SYNC JOB.md
new file mode 100644
index 0000000..4612beb
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/PAUSE SYNC JOB.md
@@ -0,0 +1,43 @@
+---
+{
+ "title": "PAUSE SYNC JOB",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# PAUSE SYNC JOB
+
+## description
+
+通过 `job_name` 暂停一个数据库内正在运行的常驻数据同步作业，被暂停的作业将停止同步数据，保持消费的最新位置，直到被用户恢复。
+
+语法：
+
+ PAUSE SYNC JOB [db.]job_name
+
+## example
+1. 暂停名称为 `job_name` 的数据同步作业。
+
+ PAUSE SYNC JOB `job_name`;
+
+## keyword
+ PAUSE,SYNC,JOB,BINLOG
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/RESUME SYNC JOB.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/RESUME SYNC JOB.md
new file mode 100644
index 0000000..9237419
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/RESUME SYNC JOB.md
@@ -0,0 +1,44 @@
+---
+{
+ "title": "RESUME SYNC JOB",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# RESUME SYNC JOB
+
+## description
+
+通过 `job_name`恢复一个当前数据库已被暂停的常驻数据同步作业，作业将从上一次被暂停前最新的位置继续同步数据。
+
+语法:
+
+ RESUME SYNC JOB [db.]job_name
+
+## example
+
+1. 恢复名称为 `job_name` 的数据同步作业
+
+ RESUME SYNC JOB `job_name`;
+
+## keyword
+ RESUME,SYNC,JOB,BINLOG
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW SYNC JOB.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW SYNC JOB.md
new file mode 100644
index 0000000..8fbde61
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW SYNC JOB.md
@@ -0,0 +1,49 @@
+---
+{
+ "title": "SHOW SYNC JOB",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# SHOW SYNC JOB
+
+## description
+
+此命令用于当前显示所有数据库内的常驻数据同步作业状态。
+
+语法：
+
+ SHOW SYNC JOB [FROM db_name]
+
+## example
+
+1. 展示当前数据库的所有数据同步作业状态。
+
+ SHOW SYNC JOB;
+
+2. 展示数据库 `test_db` 下的所有数据同步作业状态。
+
+ SHOW SYNC JOB FROM `test_db`;
+
+## keyword
+
+ SHOW,SYNC,JOB,BINLOG
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STOP SYNC JOB.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STOP SYNC JOB.md
new file mode 100644
index 0000000..cf9cbca
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STOP SYNC JOB.md
@@ -0,0 +1,44 @@
+---
+{
+ "title": "STOP SYNC JOB",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# STOP SYNC JOB
+
+## description
+
+通过 `job_name` 停止一个数据库内非停止状态的常驻数据同步作业。
+
+语法:
+
+ STOP SYNC JOB [db.]job_name
+
+## example
+
+1. 停止名称为 `job_name` 的数据同步作业
+
+ STOP SYNC JOB `job_name`;
+
+## keyword
+ STOP,SYNC,JOB,BINLOG
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 09/14: [Bug] Fix bug that BE will crash when
backup using S3 (#6855)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 986bd1abdc7a2c96b72b57afc99a39f7fa69b427
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Sun Oct 17 22:54:42 2021 +0800
[Bug] Fix bug that BE will crash when backup using S3 (#6855)
---
be/src/runtime/snapshot_loader.cpp | 6 +++---
be/src/runtime/snapshot_loader.h | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp
index 3284493..e3f5f35 100644
--- a/be/src/runtime/snapshot_loader.cpp
+++ b/be/src/runtime/snapshot_loader.cpp
@@ -49,7 +49,7 @@ SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id,
_task_id(task_id),
_broker_addr(broker_addr),
_prop(broker_prop) {
- _storage_backend.reset(new BrokerStorageBackend(_env, broker_addr, broker_prop));
+ _storage_backend.reset(new BrokerStorageBackend(_env, _broker_addr, _prop));
}
SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id)
@@ -66,10 +66,10 @@ SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id, co
_task_id(task_id),
_broker_addr(TNetworkAddress()),
_prop(prop) {
- _storage_backend.reset(new S3StorageBackend(prop));
+ _storage_backend.reset(new S3StorageBackend(_prop));
}
-SnapshotLoader::~SnapshotLoader() {}
+SnapshotLoader::~SnapshotLoader() = default;
Status SnapshotLoader::upload(const std::map<std::string, std::string>& src_to_dest_path,
std::map<int64_t, std::vector<std::string>>* tablet_files) {
diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h
index 8f4bd6c..12acc66 100644
--- a/be/src/runtime/snapshot_loader.h
+++ b/be/src/runtime/snapshot_loader.h
@@ -103,8 +103,8 @@ private:
ExecEnv* _env;
int64_t _job_id;
int64_t _task_id;
- const TNetworkAddress& _broker_addr;
- const std::map<std::string, std::string>& _prop;
+ const TNetworkAddress _broker_addr;
+ const std::map<std::string, std::string> _prop;
std::unique_ptr<StorageBackend> _storage_backend;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 10/14: [UT] Fix ut bugs (#6862)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 26e3a37f442ead2e45c46be85ded460088fd90c4
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Mon Oct 18 10:12:55 2021 +0800
[UT] Fix ut bugs (#6862)
Co-authored-by: morningman <ch...@baidu.com>
---
be/src/olap/tablet_meta.cpp | 4 ++--
be/src/olap/tablet_meta.h | 2 +-
be/test/olap/test_data/header_without_inc_rs.txt | 1 +
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 564ced8..4ede471 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -53,7 +53,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
uint32_t next_unique_id,
const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
TabletUid tablet_uid, TTabletType::type tabletType)
- : _tablet_uid(0, 0), _preferred_rowset_type(BETA_ROWSET), _schema(new TabletSchema) {
+ : _tablet_uid(0, 0), _schema(new TabletSchema) {
TabletMetaPB tablet_meta_pb;
tablet_meta_pb.set_table_id(table_id);
tablet_meta_pb.set_partition_id(partition_id);
@@ -151,10 +151,10 @@ TabletMeta::TabletMeta(const TabletMeta& b)
_tablet_type(b._tablet_type),
_tablet_state(b._tablet_state),
_schema(b._schema),
- _in_restore_mode(b._in_restore_mode),
_rs_metas(b._rs_metas),
_stale_rs_metas(b._stale_rs_metas),
_del_pred_array(b._del_pred_array),
+ _in_restore_mode(b._in_restore_mode),
_preferred_rowset_type(b._preferred_rowset_type) {
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 8995e96..069507e 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -199,7 +199,7 @@ private:
DelPredicateArray _del_pred_array;
bool _in_restore_mode = false;
- RowsetTypePB _preferred_rowset_type = ALPHA_ROWSET;
+ RowsetTypePB _preferred_rowset_type = BETA_ROWSET;
RWMutex _meta_lock;
};
diff --git a/be/test/olap/test_data/header_without_inc_rs.txt b/be/test/olap/test_data/header_without_inc_rs.txt
index 96acbb3..edd7d03 100644
--- a/be/test/olap/test_data/header_without_inc_rs.txt
+++ b/be/test/olap/test_data/header_without_inc_rs.txt
@@ -140,5 +140,6 @@
"hi": 10,
"lo": 10
},
+ "preferred_rowset_type": "BETA_ROWSET",
"tablet_type": "TABLET_TYPE_DISK"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 14/14: [ARM64] Fix some problem when compiling on
ARM64 platform (#6836) (#6872)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 259cc7b2ad35b885b3b79fe6ff4575fdab8f6442
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Tue Oct 19 13:26:02 2021 +0800
[ARM64] Fix some problem when compiling on ARM64 platform (#6836) (#6872)
With thirdparties 1.4.0 to 1.4.1
1. Add patch for aws-c-cal-0.4.5
2. Add some solutions for `undefined reference libpsl`
3. Move libgsasl to fix link problme of libcurl.
4. Downgrade openssl to 1.0.2k to fix problem of low version glibc
---
be/CMakeLists.txt | 8 ++++----
be/src/util/aes_util.cpp | 16 ++++++++--------
docs/en/installing/compilation-arm.md | 8 ++++++++
docs/en/installing/compilation.md | 8 +++++---
docs/zh-CN/installing/compilation-arm.md | 13 +++++++++++--
docs/zh-CN/installing/compilation.md | 8 +++++---
thirdparty/CHANGELOG.md | 14 ++++++++++++--
thirdparty/download-thirdparty.sh | 12 ++++++++++++
thirdparty/patches/aws-c-cal-0.4.5.patch | 11 +++++++++++
thirdparty/vars.sh | 8 ++++----
10 files changed, 80 insertions(+), 26 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index b9edcda..d8ac46d 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -294,14 +294,14 @@ set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
add_library(minzip STATIC IMPORTED)
set_target_properties(minzip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libminizip.a)
+add_library(gsasl STATIC IMPORTED)
+set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
+
if (ARCH_AMD64)
# libhdfs3 only support x86 or amd64
add_library(hdfs3 STATIC IMPORTED)
set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a)
- add_library(gsasl STATIC IMPORTED)
- set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
-
add_library(xml2 STATIC IMPORTED)
set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
@@ -494,6 +494,7 @@ set(COMMON_THIRDPARTY
lz4
libevent
libevent_pthreads
+ gsasl
curl
${LIBZ}
${LIBBZ2}
@@ -531,7 +532,6 @@ endif()
set(X86_DEPENDENCIES
${COMMON_THIRDPARTY}
hdfs3
- gsasl
xml2
lzma
)
diff --git a/be/src/util/aes_util.cpp b/be/src/util/aes_util.cpp
index 67d9fe1..1b4c7d1 100644
--- a/be/src/util/aes_util.cpp
+++ b/be/src/util/aes_util.cpp
@@ -102,6 +102,7 @@ static int do_encrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher,
int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_length,
const unsigned char* key, uint32_t key_length, const unsigned char* iv,
bool padding, unsigned char* encrypt) {
+ EVP_CIPHER_CTX aes_ctx;
const EVP_CIPHER* cipher = get_evp_type(mode);
/* The encrypt key to be used for encryption */
unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8];
@@ -110,12 +111,11 @@ int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_
if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) {
return AES_BAD_DATA;
}
- EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new();
- EVP_CIPHER_CTX_reset(aes_ctx);
+ EVP_CIPHER_CTX_init(&aes_ctx);
int length = 0;
- int ret = do_encrypt(aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt,
+ int ret = do_encrypt(&aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt,
&length);
- EVP_CIPHER_CTX_free(aes_ctx);
+ EVP_CIPHER_CTX_cleanup(&aes_ctx);
if (ret == 0) {
ERR_clear_error();
return AES_BAD_DATA;
@@ -150,6 +150,7 @@ static int do_decrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher,
int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length,
const unsigned char* key, uint32_t key_length, const unsigned char* iv,
bool padding, unsigned char* decrypt_content) {
+ EVP_CIPHER_CTX aes_ctx;
const EVP_CIPHER* cipher = get_evp_type(mode);
/* The encrypt key to be used for decryption */
@@ -160,12 +161,11 @@ int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encryp
return AES_BAD_DATA;
}
- EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new();
- EVP_CIPHER_CTX_reset(aes_ctx);
+ EVP_CIPHER_CTX_init(&aes_ctx);
int length = 0;
- int ret = do_decrypt(aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding,
+ int ret = do_decrypt(&aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding,
decrypt_content, &length);
- EVP_CIPHER_CTX_free(aes_ctx);
+ EVP_CIPHER_CTX_cleanup(&aes_ctx);
if (ret > 0) {
return length;
} else {
diff --git a/docs/en/installing/compilation-arm.md b/docs/en/installing/compilation-arm.md
index 19276d8..a013979 100644
--- a/docs/en/installing/compilation-arm.md
+++ b/docs/en/installing/compilation-arm.md
@@ -222,3 +222,11 @@ Suppose Doris source code is under `/home/doris/doris-src/`.
Execute `sh build.sh`.
+### 5. FAQ
+
+1. `undefined reference to psl_free` appears when compiling Doris
+
+ libcurl will call libpsl functions, but libpsl is not linked for an unknown reason. Solutions (choose one of the two):
+
+ 1. Add `--without-libpsl` to the `build_curl` method in `thirdparty/build-thirdparty.sh`, recompile libcurl, and then recompile Doris.
+ 2. About line 603 in `be/CMakeLists.txt`, add `-lpsl` after `-pthread`, and then recompile Doris.
diff --git a/docs/en/installing/compilation.md b/docs/en/installing/compilation.md
index f48340d..4e18c77 100644
--- a/docs/en/installing/compilation.md
+++ b/docs/en/installing/compilation.md
@@ -53,13 +53,15 @@ Note: For different versions of Doris, you need to download the corresponding mi
| apache/incubator-doris:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) or later | 0.10.x or later |
| apache/incubator-doris:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x - 0.14.0 |
| apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) or later | 0.14.x |
-| apache/incubator-doris:build-env-1.4.0 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 1.15.0 or later |
+| apache/incubator-doris:build-env-1.4.1 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 1.15.0 or later |
**note**:
-> 1. Doris version 0.14.0 still uses apache/incubator-doris:build-env-1.2 to compile, and the subsequent code will use apache/incubator-doris:build-env-1.3.1.
+> 1. Dev docker image [ChangeLog](https://github.com/apache/incubator-doris/blob/master/thirdparty/CHANGELOG.md)
-> 2. In the docker image of build-env-1.3.1, both OpenJDK 8 and OpenJDK 11 are included, and OpenJDK 11 is used for compilation by default. Please make sure that the JDK version used for compiling is the same as the JDK version used at runtime, otherwise it may cause unexpected operation errors. You can use the following command to switch the default JDK version in container:
+> 2. Doris version 0.14.0 still uses apache/incubator-doris:build-env-1.2 to compile, and the subsequent code will use apache/incubator-doris:build-env-1.3.1.
+
+> 3. In the docker image of build-env-1.3.1, both OpenJDK 8 and OpenJDK 11 are included, and OpenJDK 11 is used for compilation by default. Please make sure that the JDK version used for compiling is the same as the JDK version used at runtime, otherwise it may cause unexpected operation errors. You can use the following command to switch the default JDK version in container:
>
> Switch to JDK 8:
>
diff --git a/docs/zh-CN/installing/compilation-arm.md b/docs/zh-CN/installing/compilation-arm.md
index fbf750f..772a474 100644
--- a/docs/zh-CN/installing/compilation-arm.md
+++ b/docs/zh-CN/installing/compilation-arm.md
@@ -177,7 +177,7 @@ make -j && make install
1. 关闭 `build_mysql` 和 `build_libhdfs3`
- mysql 不在需要。而 libhdfs3 暂不支持 arm 架构，所以在arm中运行Doris，暂不支持通过 libhdfs3 直接访问 hdfs，需要通过broker。
+ mysql 不再需要。而 libhdfs3 暂不支持 arm 架构，所以在arm中运行Doris，暂不支持通过 libhdfs3 直接访问 hdfs，需要通过broker。
2. 在 `build_curl` 中增加 configure 参数：`--without-libpsl`。如果不添加，则在最终编译Doris BE的链接阶段，可能报错：`undefined reference to ‘psl_is_cookie_domain_acceptable'`
@@ -217,7 +217,16 @@ make -j && make install
* 编译到某个阶段卡住不动。
不确定原因。解决方案：重跑 `build-thirdparty.sh`。`build-thirdparty.sh` 是可以重复执行的。
-
+
### 4. 编译Doris源码
执行 `sh build.sh` 即可。
+
+### 5. 常见错误
+
+1. 编译 Doris 时出现 `undefined reference to psl_free`
+
+ libcurl 会调用 libpsl 的函数，但 libpsl 未连接，原因未知。解决方法（二选一）：
+
+ 1. 在 `thirdparty/build-thirdparty.sh` 中的 `build_curl` 方法中添加 `--without-libpsl` 后重新编译 libcurl，然后再重新编译 Doris。
+ 2. `be/CMakeLists.txt` 中 603 行左右，`-pthread` 后添加 `-lpsl`，然后重新编译 Doris。
diff --git a/docs/zh-CN/installing/compilation.md b/docs/zh-CN/installing/compilation.md
index 17e2ddd..e8213c1 100644
--- a/docs/zh-CN/installing/compilation.md
+++ b/docs/zh-CN/installing/compilation.md
@@ -52,13 +52,15 @@ under the License.
| apache/incubator-doris:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.10.x, 0.11.x |
| apache/incubator-doris:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) | 0.12.x - 0.14.0 |
| apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) | 0.14.x |
-| apache/incubator-doris:build-env-1.4.0 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 后续的发布版本 |
+| apache/incubator-doris:build-env-1.4.1 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 后续的发布版本 |
**注意**：
-> 1. doris 0.14.0 版本仍然使用apache/incubator-doris:build-env-1.2 编译，之后的代码将使用apache/incubator-doris:build-env-1.3.1。
+> 1. 编译镜像 [ChangeLog](https://github.com/apache/incubator-doris/blob/master/thirdparty/CHANGELOG.md)。
-> 2. 在 build-env-1.3.1 的docker镜像中，同时包含了 OpenJDK 8 和 OpenJDK 11，并且默认使用 OpenJDK 11 编译。请确保编译使用的 JDK 版本和运行时使用的 JDK 版本一致，否则会导致非预期的运行错误。你可以使用在进入编译镜像的容器后，使用以下命令切换默认 JDK 版本：
+> 2. doris 0.14.0 版本仍然使用apache/incubator-doris:build-env-1.2 编译，之后的代码将使用apache/incubator-doris:build-env-1.3.1。
+
+> 3. 在 build-env-1.3.1 的docker镜像中，同时包含了 OpenJDK 8 和 OpenJDK 11，并且默认使用 OpenJDK 11 编译。请确保编译使用的 JDK 版本和运行时使用的 JDK 版本一致，否则会导致非预期的运行错误。你可以使用在进入编译镜像的容器后，使用以下命令切换默认 JDK 版本：
>
> 切换到 JDK 8：
>
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 81cde3c..597a78b 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,7 +2,17 @@
This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/incubator-doris, and the tag is `build-env-${version}`
-## build-env-1.4.0 [2021-10-13]
+## build-env-1.4.1
+
+### Changes
+
+Openssl 1.1.1l introduces the getentropy() method, which requires glibc 2.25 or higher,
+which will cause Doris BE to fail to run in the low-version glibc environment.
+Temporarily roll back the openssl version.
+
+- OpenSSL 1.1.1l -> 1.0.2k
+
+## build-env-1.4.0
### Changes
@@ -135,4 +145,4 @@ This file contains version of the third-party dependency libraries in the build-
- libhdfs3: master
- libdivide: 4.0.0
- pdqsort: git20180419
-- benchmark: 1.5.6
\ No newline at end of file
+- benchmark: 1.5.6
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 8f118f0..bfe20ac 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -267,5 +267,17 @@ if [ ! -f $PATCHED_MARK ]; then
touch $PATCHED_MARK
fi
cd -
+
+# aws-c-cal patch to fix compile error
+# This bug has been fixed in new version of aws-c-cal
+if [ $AWS_C_CAL_SOURCE == "aws-c-cal-0.4.5" ]; then
+ cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE
+ if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/aws-c-cal-0.4.5.patch
+ touch $PATCHED_MARK
+ fi
+ cd -
+fi
+
echo "Finished patching $HDFS3_SOURCE"
diff --git a/thirdparty/patches/aws-c-cal-0.4.5.patch b/thirdparty/patches/aws-c-cal-0.4.5.patch
new file mode 100644
index 0000000..aca87f0
--- /dev/null
+++ b/thirdparty/patches/aws-c-cal-0.4.5.patch
@@ -0,0 +1,11 @@
+--- a/source/unix/openssl_platform_init.c 2021-10-18 19:33:56.524537840 +0800
++++ b/source/unix/openssl_platform_init.c 2021-10-18 19:35:06.534537840 +0800
+@@ -367,7 +367,7 @@ static int s_resolve_libcrypto(void) {
+ }
+
+ /* Ignore warnings about how CRYPTO_get_locking_callback() always returns NULL on 1.1.1 */
+-#if !defined(__GNUC__) || (__GNUC__ >= 4 && __GNUC_MINOR__ > 1)
++#if !defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__ * 10 > 410)
+ # pragma GCC diagnostic push
+ # pragma GCC diagnostic ignored "-Waddress"
+ #endif
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 5edd167..734a656 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -58,10 +58,10 @@ LIBEVENT_SOURCE=libevent-release-2.1.12-stable
LIBEVENT_MD5SUM="0d5a27436bf7ff8253420c8cf09f47ca"
# openssl
-OPENSSL_DOWNLOAD="https://github.com/openssl/openssl/archive/OpenSSL_1_1_1l.tar.gz"
-OPENSSL_NAME=openssl-OpenSSL_1_1_1l.tar.gz
-OPENSSL_SOURCE=openssl-OpenSSL_1_1_1l
-OPENSSL_MD5SUM="2ff02cd10f96671162df3efd60470cfe"
+OPENSSL_DOWNLOAD="https://www.openssl.org/source/old/1.0.2/openssl-1.0.2k.tar.gz"
+OPENSSL_NAME=openssl-1.0.2k.tar.gz
+OPENSSL_SOURCE=openssl-1.0.2k
+OPENSSL_MD5SUM="f965fc0bf01bf882b31314b61391ae65"
# thrift
THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/14: [Doc] Update compile docs add 0.15 build
support. (#6850)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit e386b8bc18fd6f3dd20f3a03ead8dfe10882220d
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Fri Oct 15 18:37:24 2021 +0800
[Doc] Update compile docs add 0.15 build support. (#6850)
---
docs/en/installing/compilation.md | 3 ++-
docs/zh-CN/installing/compilation.md | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/docs/en/installing/compilation.md b/docs/en/installing/compilation.md
index 3609452..f48340d 100644
--- a/docs/en/installing/compilation.md
+++ b/docs/en/installing/compilation.md
@@ -52,7 +52,8 @@ Note: For different versions of Doris, you need to download the corresponding mi
| apache/incubator-doris:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x |
| apache/incubator-doris:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) or later | 0.10.x or later |
| apache/incubator-doris:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x - 0.14.0 |
-| apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) or later | later |
+| apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) or later | 0.14.x |
+| apache/incubator-doris:build-env-1.4.0 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 1.15.0 or later |
**note**:
diff --git a/docs/zh-CN/installing/compilation.md b/docs/zh-CN/installing/compilation.md
index 6847907..17e2ddd 100644
--- a/docs/zh-CN/installing/compilation.md
+++ b/docs/zh-CN/installing/compilation.md
@@ -51,7 +51,8 @@ under the License.
| apache/incubator-doris:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x |
| apache/incubator-doris:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.10.x, 0.11.x |
| apache/incubator-doris:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) | 0.12.x - 0.14.0 |
-| apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) | 后续的发布版本 |
+| apache/incubator-doris:build-env-1.3.1 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) | 0.14.x |
+| apache/incubator-doris:build-env-1.4.0 | [24d3861](https://github.com/apache/incubator-doris/commit/24d38614a0f21ed606462816a262c2e1d8273ace) or later | 后续的发布版本 |
**注意**：
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 04/14: [SQL Cache] Add all view stmt as the
suffix of cache sqlkey (#6832)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 171ee71caa731f522e3651fa766152030d631df7
Author: Zeno Yang <co...@qq.com>
AuthorDate: Sat Oct 16 21:56:24 2021 +0800
[SQL Cache] Add all view stmt as the suffix of cache sqlkey (#6832)
Use all view stmt as the cache sqlkey suffix, so that when the view is modified, the cache can recognize.
---
.../org/apache/doris/analysis/InlineViewRef.java | 8 ++++
.../main/java/org/apache/doris/qe/cache/Cache.java | 1 +
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 43 +++++++++++++++++++++-
.../org/apache/doris/qe/cache/PartitionCache.java | 7 +++-
.../java/org/apache/doris/qe/cache/SqlCache.java | 6 ++-
5 files changed, 59 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
index 807949e..874f020 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
@@ -417,6 +417,14 @@ public class InlineViewRef extends TableRef {
return baseTblSmap;
}
+ public boolean isLocalView() {
+ return view == null || view.isLocalView();
+ }
+
+ public View getView() {
+ return view;
+ }
+
@Override
public String tableRefToSql() {
// Enclose the alias in quotes if Hive cannot parse it without quotes.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
index 1711179..6d80556 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
@@ -43,6 +43,7 @@ public abstract class Cache {
protected CacheAnalyzer.CacheTable latestTable;
protected CacheProxy proxy;
protected HitRange hitRange;
+ protected String allViewExpandStmtListStr;
protected Cache(TUniqueId queryId, SelectStmt selectStmt) {
this.queryId = queryId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
index d8a6f47..e651d00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
@@ -17,6 +17,7 @@
package org.apache.doris.qe.cache;
+import org.apache.commons.lang3.StringUtils;
import org.apache.doris.analysis.AggregateInfo;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.CastExpr;
@@ -25,6 +26,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.InlineViewRef;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SelectStmt;
+import org.apache.doris.analysis.SetOperationStmt;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TableRef;
@@ -33,6 +35,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.RangePartitionInfo;
+import org.apache.doris.catalog.View;
import org.apache.doris.common.Config;
import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
@@ -51,8 +54,11 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* Analyze which caching mode a SQL is suitable for
@@ -87,6 +93,7 @@ public class CacheAnalyzer {
private Column partColumn;
private CompoundPredicate partitionPredicate;
private Cache cache;
+ private Set<String> allViewStmtSet;
public Cache getCache() {
return cache;
@@ -98,6 +105,7 @@ public class CacheAnalyzer {
this.parsedStmt = parsedStmt;
scanNodes = planner.getScanNodes();
latestTable = new CacheTable();
+ allViewStmtSet = new HashSet<>();
checkCacheConfig();
}
@@ -106,6 +114,7 @@ public class CacheAnalyzer {
this.context = context;
this.parsedStmt = parsedStmt;
this.scanNodes = scanNodes;
+ allViewStmtSet = new HashSet<>();
checkCacheConfig();
}
@@ -209,6 +218,9 @@ public class CacheAnalyzer {
latestTable = tblTimeList.get(0);
latestTable.Debug();
+ addAllViewStmt(selectStmt);
+ String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, ",");
+
if (now == 0) {
now = nowtime();
}
@@ -216,7 +228,7 @@ public class CacheAnalyzer {
(now - latestTable.latestTime) >= Config.cache_last_version_interval_second * 1000) {
LOG.debug("TIME:{},{},{}", now, latestTable.latestTime, Config.cache_last_version_interval_second*1000);
cache = new SqlCache(this.queryId, this.selectStmt);
- ((SqlCache) cache).setCacheInfo(this.latestTable);
+ ((SqlCache) cache).setCacheInfo(this.latestTable, allViewExpandStmtListStr);
MetricRepo.COUNTER_CACHE_MODE_SQL.increase(1L);
return CacheMode.Sql;
}
@@ -263,7 +275,7 @@ public class CacheAnalyzer {
partitionPredicate = compoundPredicates.get(0);
cache = new PartitionCache(this.queryId, this.selectStmt);
((PartitionCache) cache).setCacheInfo(this.latestTable, this.partitionInfo, this.partColumn,
- this.partitionPredicate);
+ this.partitionPredicate, allViewExpandStmtListStr);
MetricRepo.COUNTER_CACHE_MODE_PARTITION.increase(1L);
return CacheMode.Partition;
}
@@ -430,6 +442,33 @@ public class CacheAnalyzer {
return table;
}
+ private void addAllViewStmt(List<TableRef> tblRefs) {
+ for (TableRef tblRef : tblRefs) {
+ if (tblRef instanceof InlineViewRef) {
+ InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
+ if (inlineViewRef.isLocalView()) {
+ Collection<View> views = inlineViewRef.getAnalyzer().getLocalViews().values();
+ for (View view : views) {
+ addAllViewStmt(view.getQueryStmt());
+ }
+ } else {
+ addAllViewStmt(inlineViewRef.getViewStmt());
+ allViewStmtSet.add(inlineViewRef.getView().getInlineViewDef());
+ }
+ }
+ }
+ }
+
+ private void addAllViewStmt(QueryStmt queryStmt) {
+ if (queryStmt instanceof SelectStmt) {
+ addAllViewStmt(((SelectStmt) queryStmt).getTableRefs());
+ } else if (queryStmt instanceof SetOperationStmt) {
+ for (SetOperationStmt.SetOperand operand : ((SetOperationStmt) queryStmt).getOperands()) {
+ addAllViewStmt(((SelectStmt) operand.getQueryStmt()).getTableRefs());
+ }
+ }
+ }
+
public Cache.HitRange getHitRange() {
if (cacheMode == CacheMode.None) {
return Cache.HitRange.None;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
index d208d77..1ff1fdb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
@@ -66,13 +66,14 @@ public class PartitionCache extends Cache {
}
public void setCacheInfo(CacheAnalyzer.CacheTable latestTable, RangePartitionInfo partitionInfo, Column partColumn,
- CompoundPredicate partitionPredicate) {
+ CompoundPredicate partitionPredicate, String allViewExpandStmtListStr) {
this.latestTable = latestTable;
this.olapTable = latestTable.olapTable;
this.partitionInfo = partitionInfo;
this.partColumn = partColumn;
this.partitionPredicate = partitionPredicate;
this.newRangeList = Lists.newArrayList();
+ this.allViewExpandStmtListStr = allViewExpandStmtListStr;
}
public InternalService.PFetchCacheResult getCacheData(Status status) {
@@ -84,8 +85,10 @@ public class PartitionCache extends Cache {
status.setStatus("analytics range error");
return null;
}
+
+ String nokeyStmtWithViewStmt = nokeyStmt.toSql() + allViewExpandStmtListStr;
InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder()
- .setSqlKey(CacheProxy.getMd5(nokeyStmt.toSql()))
+ .setSqlKey(CacheProxy.getMd5(nokeyStmtWithViewStmt))
.addAllParams(range.getPartitionSingleList().stream().map(
p -> InternalService.PCacheParam.newBuilder()
.setPartitionKey(p.getCacheKey().realValue())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index 5d1d6d9..9485547 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -35,13 +35,15 @@ public class SqlCache extends Cache {
super(queryId, selectStmt);
}
- public void setCacheInfo(CacheAnalyzer.CacheTable latestTable) {
+ public void setCacheInfo(CacheAnalyzer.CacheTable latestTable, String allViewExpandStmtListStr) {
this.latestTable = latestTable;
+ this.allViewExpandStmtListStr = allViewExpandStmtListStr;
}
public InternalService.PFetchCacheResult getCacheData(Status status) {
+ String originStmtWithViewStmt = selectStmt.getOrigStmt().originStmt + allViewExpandStmtListStr;
InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder()
- .setSqlKey(CacheProxy.getMd5(selectStmt.getOrigStmt().originStmt))
+ .setSqlKey(CacheProxy.getMd5(originStmtWithViewStmt))
.addParams(InternalService.PCacheParam.newBuilder()
.setPartitionKey(latestTable.latestPartitionId)
.setLastVersion(latestTable.latestVersion)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/14: [BUG] Key is 'True',
Extra is 'NONE' when add rollup for DUP table (#6763)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 59ce1353939a7dd8e2e4163b7c948b11ec657d95
Author: shee <13...@users.noreply.github.com>
AuthorDate: Sat Oct 16 21:54:00 2021 +0800
[BUG] Key is 'True',Extra is 'NONE' when add rollup for DUP table (#6763)
fix #6762
The result is displayed after the repair
```
mysql> desc test2 all;
+-----------+---------------+-------+---------------+------+-------+---------+-------+---------+
| IndexName | IndexKeysType | Field | Type | Null | Key | Default | Extra | Visible |
+-----------+---------------+-------+---------------+------+-------+---------+-------+---------+
| test2 | DUP_KEYS | a | BIGINT | Yes | true | NULL | | true |
| | | b | BIGINT | Yes | true | NULL | | true |
| | | c | BIGINT | Yes | false | NULL | NONE | true |
| | | d | BIGINT | Yes | false | NULL | NONE | true |
| | | e | VARCHAR(1024) | Yes | false | NULL | NONE | true |
| | | f | VARCHAR(1024) | Yes | false | NULL | NONE | true |
| | | | | | | | | |
| r1 | DUP_KEYS | c | BIGINT | Yes | true | NULL | | true |
| | | e | VARCHAR(1024) | Yes | true | NULL | | true |
| | | a | BIGINT | Yes | false | NULL | NONE | true |
+-----------+---------------+-------+---------------+------+-------+---------+-------+---------+
```
---
.../main/java/org/apache/doris/catalog/Column.java | 4 ++
.../org/apache/doris/alter/RollupJobV2Test.java | 20 +++++++
.../org/apache/doris/catalog/CatalogTestUtil.java | 66 +++++++++++++++++++++-
3 files changed, 89 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index c182377..b875563 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -201,6 +201,10 @@ public class Column implements Writable {
}
public void setIsKey(boolean isKey) {
+ // column is key, aggregationType is always null, isAggregationTypeImplicit is always false.
+ if (isKey) {
+ setAggregationType(null, false);
+ }
this.isKey = isKey;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
index 8a6f8f5..81db108 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
@@ -426,4 +426,24 @@ public class RollupJobV2Test {
Assert.assertEquals("to_bitmap", resultFunctionCall.getFnName().getFunction());
}
+
+ @Test
+ public void testAddRollupForDupTable() throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ MaterializedViewHandler materializedViewHandler = Catalog.getCurrentCatalog().getRollupHandler();
+ Database db = masterCatalog.getDbOrDdlException(CatalogTestUtil.testDbId1);
+ OlapTable olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+
+ AddRollupClause addRollupClause = new AddRollupClause("r1", Lists.newArrayList("k1", "v1", "v2"), null, CatalogTestUtil.testIndex2, null);
+
+ List<Column> columns = materializedViewHandler.checkAndPrepareMaterializedView(addRollupClause, olapTable, CatalogTestUtil.testIndexId2, false);
+ for (Column column : columns) {
+ if (column.nameEquals("v1", true)) {
+ Assert.assertNull(column.getAggregationType());
+ break;
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
index b540ec6..93eaa19 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
@@ -47,12 +47,23 @@ public class CatalogTestUtil {
public static String testDb1 = "testDb1";
public static long testDbId1 = 1;
public static String testTable1 = "testTable1";
+ public static String testTable2 = "testTable2";
+
public static long testTableId1 = 2;
+ public static long testTableId2 = 15;
+
public static String testPartition1 = "testPartition1";
+ public static String testPartition2 = "testPartition2";
+
public static long testPartitionId1 = 3;
+ public static long testPartitionId2 = 16;
+
public static String testIndex1 = "testIndex1";
public static String testIndex2 = "testIndex2";
+
public static long testIndexId1 = 2; // the base indexId == tableId
+ public static long testIndexId2 = 15; // the base indexId == tableId
+
public static int testSchemaHash1 = 93423942;
public static long testBackendId1 = 5;
public static long testBackendId2 = 6;
@@ -60,7 +71,11 @@ public class CatalogTestUtil {
public static long testReplicaId1 = 8;
public static long testReplicaId2 = 9;
public static long testReplicaId3 = 10;
+ public static long testReplicaId4 = 17;
+
public static long testTabletId1 = 11;
+ public static long testTabletId2 = 18;
+
public static long testStartVersion = 12;
public static long testStartVersionHash = 12312;
public static long testPartitionCurrentVersionHash = 12312;
@@ -230,13 +245,62 @@ public class CatalogTestUtil {
// add a es table to catalog
try {
createEsTable(db);
+ createDupTable(db);
} catch (DdlException e) {
// TODO Auto-generated catch block
// e.printStackTrace();
}
return db;
}
-
+
+ public static void createDupTable(Database db) {
+
+ // replica
+ Replica replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, testStartVersionHash, 0, 0L, 0L,
+ ReplicaState.NORMAL, -1, 0, 0, 0);
+
+ // tablet
+ Tablet tablet = new Tablet(testTabletId2);
+
+ // index
+ MaterializedIndex index = new MaterializedIndex(testIndexId2, IndexState.NORMAL);
+ TabletMeta tabletMeta = new TabletMeta(testDbId1, testTableId2, testPartitionId2, testIndexId2, 0, TStorageMedium.HDD);
+ index.addTablet(tablet, tabletMeta);
+
+ tablet.addReplica(replica);
+
+ // partition
+ RandomDistributionInfo distributionInfo = new RandomDistributionInfo(1);
+ Partition partition = new Partition(testPartitionId2, testPartition2, index, distributionInfo);
+ partition.updateVisibleVersionAndVersionHash(testStartVersion, testStartVersionHash);
+ partition.setNextVersion(testStartVersion + 1);
+ partition.setNextVersionHash(testPartitionNextVersionHash, testPartitionCurrentVersionHash);
+
+ // columns
+ List<Column> columns = new ArrayList<Column>();
+ Column temp = new Column("k1", PrimitiveType.INT);
+ temp.setIsKey(true);
+ columns.add(temp);
+ temp = new Column("k2", PrimitiveType.INT);
+ temp.setIsKey(true);
+ columns.add(temp);
+ columns.add(new Column("v1", ScalarType.createType(PrimitiveType.VARCHAR), false, AggregateType.NONE, "0", ""));
+ columns.add(new Column("v2", ScalarType.createType(PrimitiveType.VARCHAR), false, AggregateType.NONE, "0", ""));
+
+ // table
+ PartitionInfo partitionInfo = new SinglePartitionInfo();
+ partitionInfo.setDataProperty(testPartitionId2, DataProperty.DEFAULT_DATA_PROPERTY);
+ partitionInfo.setReplicaAllocation(testPartitionId2, new ReplicaAllocation((short) 1));
+ OlapTable table = new OlapTable(testTableId2, testTable2, columns, KeysType.DUP_KEYS, partitionInfo,
+ distributionInfo);
+ table.addPartition(partition);
+ table.setIndexMeta(testIndexId2, testIndex2, columns, 0, testSchemaHash1, (short) 3,
+ TStorageType.COLUMN, KeysType.DUP_KEYS);
+ table.setBaseIndexId(testIndexId2);
+ // db
+ db.createTable(table);
+ }
+
public static void createEsTable(Database db) throws DdlException {
// columns
List<Column> columns = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org