You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by al...@apache.org on 2021/11/22 12:17:11 UTC
[incubator-inlong] branch master updated: [INLONG-1753][Feature][InLong-Manager] Add basic management APIs for Pulsar (#1818)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4cb359a [INLONG-1753][Feature][InLong-Manager] Add basic management APIs for Pulsar (#1818)
4cb359a is described below
commit 4cb359ab425a4bf0241021e7a81441588687c90d
Author: healchow <he...@gmail.com>
AuthorDate: Mon Nov 22 20:17:04 2021 +0800
[INLONG-1753][Feature][InLong-Manager] Add basic management APIs for Pulsar (#1818)
* [INLONG-1753][Feature][InLong-Manager] Add basic management APIs for Pulsar
* [INLONG-1753][Feature][InLong-Manager] Add basic management APIs for Pulsar
* [INLONG-1753][Feature][InLong-Manager] Add basic management APIs for Pulsar
* [INLONG-1753][Feature][InLong-Manager] Add UTs for Pulsar APIs
Co-authored-by: healchow <he...@gmail.com>
---
.../docker-compose/sql/apache_inlong_manager.sql | 406 +++++++-------
inlong-manager/doc/sql/apache_inlong_manager.sql | 400 +++++++-------
.../inlong/manager/common/beans/ClusterBean.java | 6 +
.../beans/{TryBean.java => ReTryConfigBean.java} | 2 +-
.../inlong/manager/common/enums/BizConstant.java | 12 +-
.../manager/common/exceptions/BaseException.java | 4 +
.../common/exceptions/BusinessException.java | 4 +
.../common/pojo/business/BusinessApproveInfo.java | 30 +
.../manager/common/pojo/business/BusinessInfo.java | 15 +-
...nessApproveInfo.java => BusinessMqExtBase.java} | 25 +-
.../common/pojo/business/BusinessPulsarInfo.java | 66 +++
.../common/pojo/business/BusinessTopicVO.java | 18 +-
.../pojo/dataconsumption/ConsumptionInfo.java | 2 +-
.../pojo/dataconsumption/ConsumptionListVo.java | 2 +-
.../pojo/dataconsumption/ConsumptionQuery.java | 2 +-
.../common/pojo/datastorage/BaseStorageInfo.java | 2 +-
.../common/pojo/datastorage/StorageHiveInfo.java | 2 +-
.../DataStreamTopicVO.java} | 23 +-
.../manager/dao/entity/BusinessPulsarEntity.java} | 28 +-
.../dao/mapper/BusinessPulsarEntityMapper.java} | 38 +-
.../manager/dao/mapper/DataStreamEntityMapper.java | 7 +-
.../mappers/BusinessPulsarEntityMapper.xml | 251 +++++++++
.../resources/mappers/DataStreamEntityMapper.xml | 8 +
.../test/resources/sql/apache_inlong_manager.sql | 612 +++++++++++----------
.../manager/service/core/DataStreamService.java | 6 +
.../service/core/impl/BusinessServiceImpl.java | 143 ++++-
.../service/core/impl/DataStreamServiceImpl.java | 47 +-
.../service/core/impl/StorageHiveOperation.java | 8 +-
.../service/core/impl/StorageServiceImpl.java | 16 +-
...va => CreateTubeConsumerGroupTaskListener.java} | 12 +-
...tener.java => CreateTubeTopicTaskListener.java} | 2 +-
...stener.java => PushHiveConfigTaskListener.java} | 4 +-
.../CreateResourceWorkflowDefinition.java | 24 +-
.../ConsumptionCompleteProcessListener.java | 2 +-
.../newstream/SingleStreamWorkflowDefinition.java | 8 +-
.../web/controller/ConsumptionController.java | 14 +-
.../src/main/resources/application-dev.properties | 5 +
.../src/main/resources/application-prod.properties | 5 +
.../src/main/resources/application-test.properties | 11 +-
.../manager/service/core/BusinessServiceTest.java | 69 +++
.../src/test/resources/application-test.properties | 5 +
.../test/resources/sql/apache_inlong_manager.sql | 602 ++++++++++----------
42 files changed, 1808 insertions(+), 1140 deletions(-)
diff --git a/docker/docker-compose/sql/apache_inlong_manager.sql b/docker/docker-compose/sql/apache_inlong_manager.sql
index e1ee1f7..6df8bdc 100644
--- a/docker/docker-compose/sql/apache_inlong_manager.sql
+++ b/docker/docker-compose/sql/apache_inlong_manager.sql
@@ -33,7 +33,7 @@ CREATE TABLE `agent_heartbeat_log`
`ip` varchar(64) NOT NULL COMMENT 'agent host ip',
`version` varchar(128) DEFAULT NULL,
`heartbeat_msg` text DEFAULT NULL COMMENT 'massage in heartbeat request',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`ip`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='Agent heartbeat information table';
@@ -65,7 +65,7 @@ CREATE TABLE `agent_sys_conf`
`stat_interval_sec` int(11) NOT NULL DEFAULT '60' COMMENT 'Statistical message sending frequency',
`conf_refresh_interval_secs` int(11) NOT NULL DEFAULT '300' COMMENT 'The frequency at which the Agent regularly pulls the configuration from the InLongManager',
`flow_size` int(11) DEFAULT '1048576000',
- `bufferSize` int(11) DEFAULT NULL COMMENT 'bufferSize, default 1048576',
+ `bufferSize` int(11) DEFAULT '1048576' COMMENT 'bufferSize, default 1048576',
`compress` tinyint(2) DEFAULT NULL COMMENT 'Whether to compress',
`event_check_interval` int(11) DEFAULT NULL COMMENT 'File scanning period',
`is_calMD5` tinyint(2) DEFAULT NULL COMMENT 'Do you want to calculate the cumulative md5 of read characters',
@@ -83,30 +83,53 @@ CREATE TABLE `business`
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
`name` varchar(128) DEFAULT '' COMMENT 'Business name, English, numbers and underscore',
`cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'Business Introduction',
- `middleware_type` varchar(10) DEFAULT 'Tube' COMMENT 'The middleware type of data storage, high throughput: Tube',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, for Tube, its Topic',
- `daily_records` int(11) DEFAULT NULL COMMENT 'Number of access records per day, unit: 10,000 records per day',
- `daily_storage` int(11) DEFAULT NULL COMMENT 'Access size by day, unit: GB per day',
- `peak_records` int(11) DEFAULT NULL COMMENT 'Access peak per second, unit: records per second',
- `max_length` int(11) DEFAULT NULL COMMENT 'The maximum length of a single piece of data, unit: Byte',
+ `description` varchar(256) DEFAULT '' COMMENT 'Business Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `mq_resource_obj` varchar(128) NOT NULL COMMENT 'MQ resource object, for Tube, its Topic, for Pulsar, its Namespace',
+ `daily_records` int(11) DEFAULT '10' COMMENT 'Number of access records per day, unit: 10,000 records per day',
+ `daily_storage` int(11) DEFAULT '10' COMMENT 'Access size by day, unit: GB per day',
+ `peak_records` int(11) DEFAULT '1000' COMMENT 'Access peak per second, unit: records per second',
+ `max_length` int(11) DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte',
`schema_name` varchar(128) DEFAULT NULL COMMENT 'Data type, associated data_schema table',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
`followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
`status` int(11) DEFAULT '21' COMMENT 'Business status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_business` (`inlong_group_id`, `is_deleted`, `modify_time`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='Business table';
-- ----------------------------
+-- Table structure for business_pulsar
+-- ----------------------------
+DROP TABLE IF EXISTS `business_pulsar`;
+CREATE TABLE `business_pulsar`
+(
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
+ `ensemble` int(3) DEFAULT '3' COMMENT 'The writable nodes number of ledger',
+ `write_quorum` int(3) DEFAULT '3' COMMENT 'The copies number of ledger',
+ `ack_quorum` int(3) DEFAULT '2' COMMENT 'The number of requested acks',
+ `retention_time` int(11) DEFAULT '72' COMMENT 'Message storage time',
+ `retention_time_unit` char(20) DEFAULT 'hours' COMMENT 'The unit of the message storage time',
+ `ttl` int(11) DEFAULT '24' COMMENT 'Message time-to-live duration',
+ `ttl_unit` varchar(20) DEFAULT 'hours' COMMENT 'The unit of time-to-live duration',
+ `retention_size` int(11) DEFAULT '-1' COMMENT 'Message size',
+ `retention_size_unit` varchar(20) DEFAULT 'MB' COMMENT 'The unit of message size',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ PRIMARY KEY (`id`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8mb4 COMMENT ='Pulsar info table';
+
+-- ----------------------------
-- Table structure for business_ext
-- ----------------------------
DROP TABLE IF EXISTS `business_ext`;
@@ -117,9 +140,9 @@ CREATE TABLE `business_ext`
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
`key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
- KEY `index_bid` (`inlong_group_id`)
+ KEY `index_group_id` (`inlong_group_id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='Business extension table';
@@ -131,19 +154,19 @@ CREATE TABLE `cluster_info`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`name` varchar(128) NOT NULL COMMENT 'cluster name',
- `type` varchar(32) DEFAULT NULL COMMENT 'Cluster type, including TUBE, ZOOKEEPER, etc.',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Cluster IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Cluster port number',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `type` varchar(32) NOT NULL COMMENT 'Cluster type, including TUBE, PULSAR, etc.',
+ `ip` varchar(64) NOT NULL COMMENT 'Cluster IP address',
+ `port` int(11) NOT NULL COMMENT 'Cluster port',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
`url` varchar(256) DEFAULT NULL COMMENT 'Cluster URL address',
`is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
`ext_props` json DEFAULT NULL COMMENT 'extended properties',
`status` int(11) DEFAULT '1' COMMENT 'cluster status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='Cluster Information Table';
@@ -154,35 +177,35 @@ CREATE TABLE `cluster_info`
DROP TABLE IF EXISTS `common_db_server`;
CREATE TABLE `common_db_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
- `db_type` varchar(128) DEFAULT NULL COMMENT 'DB type, such as MySQL, Oracle',
- `db_server_ip` varchar(64) DEFAULT NULL COMMENT 'DB Server IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `username` varchar(64) DEFAULT NULL COMMENT 'Username',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
- `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
- `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
- `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
- `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
- `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
- `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
- `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `connection_name` varchar(128) NOT NULL COMMENT 'The name of the database connection',
+ `db_type` varchar(128) DEFAULT 'MySQL' COMMENT 'DB type, such as MySQL, Oracle',
+ `db_server_ip` varchar(64) NOT NULL COMMENT 'DB Server IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
+ `username` varchar(64) NOT NULL COMMENT 'Username',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
+ `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
+ `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
+ `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
+ `in_charges` varchar(512) NOT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
+ `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
+ `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
+ `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
+ `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- DEFAULT CHARSET = utf8mb4 COMMENT ='public DB data source';
+ DEFAULT CHARSET = utf8mb4 COMMENT ='Common Database Server Table';
-- ----------------------------
-- Table structure for common_file_server
@@ -190,25 +213,25 @@ CREATE TABLE `common_db_server`
DROP TABLE IF EXISTS `common_file_server`;
CREATE TABLE `common_file_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Data source IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
- `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
- `username` varchar(64) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy, LoadProxy',
+ `ip` varchar(64) NOT NULL COMMENT 'Data source IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
+ `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
+ `username` varchar(64) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- DEFAULT CHARSET = utf8mb4 COMMENT ='public file data source';
+ DEFAULT CHARSET = utf8mb4 COMMENT ='Common File Server Table';
-- ----------------------------
-- Table structure for consumption
@@ -219,22 +242,20 @@ CREATE TABLE `consumption`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`consumer_group_name` varchar(255) NOT NULL COMMENT 'consumer group name',
`consumer_group_id` varchar(255) NOT NULL COMMENT 'Consumer group ID',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Person in charge of consumption',
- `inlong_group_id` varchar(255) DEFAULT NULL COMMENT 'Business group id',
- `middleware_type` varchar(64) DEFAULT NULL COMMENT 'The middleware type of data storage, high throughput: Tube',
- `topic` varchar(255) DEFAULT NULL COMMENT 'Consumption topic',
- `filter_enabled` int(2) DEFAULT NULL COMMENT 'Whether to filter',
- `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Data stream ID for consumption',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Person in charge of consumption',
+ `inlong_group_id` varchar(255) NOT NULL COMMENT 'Business group id',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `topic` varchar(255) NOT NULL COMMENT 'Consumption topic',
+ `filter_enabled` int(2) DEFAULT '0' COMMENT 'Whether to filter, default 0, not filter consume',
+ `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Data stream ID for consumption, if filter_enable is 1, it cannot empty',
`status` int(11) NOT NULL COMMENT 'Status: draft, pending approval, approval rejected, approval passed',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator',
+ `creator` varchar(64) NOT NULL COMMENT 'creator',
`modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Modify time',
`is_deleted` int(2) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- PRIMARY KEY (`id`),
- UNIQUE KEY `consumer_group_id_is_deleted_uindex` (`consumer_group_id`, `is_deleted`)
+ PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 6
DEFAULT CHARSET = utf8mb4 COMMENT ='Data consumption configuration table';
-- ----------------------------
@@ -243,22 +264,22 @@ CREATE TABLE `consumption`
DROP TABLE IF EXISTS `data_proxy_cluster`;
CREATE TABLE `data_proxy_cluster`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `name` varchar(128) DEFAULT NULL COMMENT 'cluster name',
- `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
- `address` varchar(128) DEFAULT NULL COMMENT 'cluster address',
- `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
- `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
- `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `ext_props` json DEFAULT NULL COMMENT 'Extended properties',
- `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'Create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `name` varchar(128) NOT NULL COMMENT 'cluster name',
+ `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
+ `address` varchar(128) NOT NULL COMMENT 'cluster address',
+ `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
+ `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
+ `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
+ `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `ext_props` json DEFAULT NULL COMMENT 'Extended properties',
+ `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='DataProxy cluster table';
@@ -278,7 +299,6 @@ CREATE TABLE `data_schema`
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 10
DEFAULT CHARSET = utf8mb4 COMMENT ='Data format table';
-- create default data schema
@@ -297,7 +317,7 @@ CREATE TABLE `data_source_cmd_config`
`specified_data_time` varchar(64) NOT NULL,
`bSend` tinyint(1) NOT NULL,
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Last update time ',
- `create_time` timestamp NULL DEFAULT NULL,
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`result_info` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `index_1` (`task_id`, `bSend`, `specified_data_time`)
@@ -313,9 +333,9 @@ CREATE TABLE `data_stream`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id, non-deleted globally unique',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
- `name` varchar(64) DEFAULT NULL COMMENT 'The name of the data stream page display, can be Chinese',
+ `name` varchar(64) NOT NULL COMMENT 'The name of the data stream page display, can be Chinese',
`description` varchar(256) DEFAULT '' COMMENT 'Introduction to data stream',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id',
+ `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id, Pulsar is Topic',
`data_source_type` varchar(32) DEFAULT 'FILE' COMMENT 'Data source type, including: FILE, DB, Auto-Push (DATA_PROXY_SDK, HTTP)',
`storage_period` int(11) DEFAULT '1' COMMENT 'The storage period of data in MQ, unit: day',
`data_type` varchar(20) DEFAULT 'TEXT' COMMENT 'Data type, there are: TEXT, KEY-VALUE, PB, BON, TEXT and BON should be treated differently',
@@ -326,15 +346,14 @@ CREATE TABLE `data_stream`
`status` int(11) DEFAULT '0' COMMENT 'Data stream status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_data_stream` (`inlong_stream_id`, `inlong_group_id`, `is_deleted`, `modify_time`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 28
DEFAULT CHARSET = utf8mb4 COMMENT ='Data stream table';
-- ----------------------------
@@ -349,9 +368,9 @@ CREATE TABLE `data_stream_ext`
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
`key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
- KEY `index_bid` (`inlong_stream_id`)
+ KEY `index_stream_id` (`inlong_stream_id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='Data stream extension table';
@@ -365,10 +384,10 @@ CREATE TABLE `data_stream_field`
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
`is_predefined_field` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a predefined field, 0: no, 1: yes',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
`field_value` varchar(128) DEFAULT NULL COMMENT 'Field value, required if it is a predefined field',
`pre_expression` varchar(256) DEFAULT NULL COMMENT 'Pre-defined field value expression',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
`field_comment` varchar(50) DEFAULT NULL COMMENT 'Field description',
`rank_num` smallint(6) DEFAULT '0' COMMENT 'Field order (front-end display field order)',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
@@ -379,7 +398,6 @@ CREATE TABLE `data_stream_field`
PRIMARY KEY (`id`),
KEY `index_stream_id` (`inlong_stream_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 39
DEFAULT CHARSET = utf8mb4 COMMENT ='File/DB data source field table';
-- ----------------------------
@@ -405,7 +423,6 @@ CREATE TABLE `operation_log`
`err_msg` text COMMENT 'Error message',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 289
DEFAULT CHARSET = utf8mb4;
-- ----------------------------
@@ -415,8 +432,8 @@ DROP TABLE IF EXISTS `role`;
CREATE TABLE `role`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
- `role_code` varchar(100) NOT NULL COMMENT 'role code',
- `role_name` varchar(255) NOT NULL COMMENT 'role Chinese name',
+ `role_code` varchar(100) NOT NULL COMMENT 'Role code',
+ `role_name` varchar(255) NOT NULL COMMENT 'Role Chinese name',
`create_time` datetime NOT NULL,
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_by` varchar(255) NOT NULL,
@@ -439,10 +456,10 @@ CREATE TABLE `source_db_basic`
`inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
`sync_type` tinyint(1) DEFAULT '0' COMMENT 'Data synchronization type, 0: FULL, full amount, 1: INCREMENTAL, incremental',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
@@ -457,7 +474,7 @@ CREATE TABLE `source_db_detail`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
`db_name` varchar(128) DEFAULT NULL COMMENT 'database name',
`transfer_ip` varchar(64) DEFAULT NULL COMMENT 'Transfer IP',
`connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
@@ -468,11 +485,11 @@ CREATE TABLE `source_db_detail`
`status` int(11) DEFAULT '0' COMMENT 'Data source status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save unsubmitted and unapproved intermediate data after modification',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='DB data source details table';
@@ -494,14 +511,13 @@ CREATE TABLE `source_file_basic`
`upload_max_size` int(4) DEFAULT '120' COMMENT 'Upload maximum size',
`need_compress` tinyint(1) DEFAULT '0' COMMENT 'Whether need compress',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Delete switch',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator',
`modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'temp view',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 9
DEFAULT CHARSET = utf8mb4 COMMENT ='basic configuration of file data source';
-- ----------------------------
@@ -514,25 +530,24 @@ CREATE TABLE `source_file_detail`
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
`access_type` varchar(20) DEFAULT 'Agent' COMMENT 'Collection type, there are Agent, DataProxy client, LoadProxy, the file can only be Agent temporarily',
- `server_name` varchar(64) DEFAULT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
- `ip` varchar(128) DEFAULT NULL COMMENT 'Data source IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Data source port number',
+ `server_name` varchar(64) NOT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
+ `ip` varchar(128) NOT NULL COMMENT 'Data source IP address',
+ `port` int(11) NOT NULL COMMENT 'Data source port number',
`is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
`issue_type` varchar(10) DEFAULT 'SSH' COMMENT 'Issuing method, there are SSH, TCS',
- `username` varchar(32) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `file_path` varchar(256) DEFAULT NULL COMMENT 'File path, supports regular matching',
+ `username` varchar(32) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `file_path` varchar(256) NOT NULL COMMENT 'File path, supports regular matching',
`status` int(11) DEFAULT '0' COMMENT 'Data source status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
`modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 3
DEFAULT CHARSET = utf8mb4 COMMENT ='Detailed table of file data source';
-- ----------------------------
@@ -547,11 +562,10 @@ CREATE TABLE `storage_ext`
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
`key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
- KEY `index_bid` (`storage_id`)
+ KEY `index_storage_id` (`storage_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 11
DEFAULT CHARSET = utf8mb4 COMMENT ='Data storage extension table';
-- ----------------------------
@@ -564,11 +578,11 @@ CREATE TABLE `storage_hive`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `jdbc_url` varchar(255) DEFAULT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
- `username` varchar(128) DEFAULT NULL COMMENT 'Username',
- `password` varchar(255) DEFAULT NULL COMMENT 'User password',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `table_name` varchar(128) DEFAULT NULL COMMENT 'Target data table name',
+ `jdbc_url` varchar(255) NOT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
+ `username` varchar(128) NOT NULL COMMENT 'Username',
+ `password` varchar(255) NOT NULL COMMENT 'User password',
+ `db_name` varchar(128) NOT NULL COMMENT 'Target database name',
+ `table_name` varchar(128) NOT NULL COMMENT 'Target data table name',
`primary_partition` varchar(255) DEFAULT 'dt' COMMENT 'primary partition field',
`secondary_partition` varchar(256) DEFAULT NULL COMMENT 'secondary partition field',
`partition_type` varchar(10) DEFAULT NULL COMMENT 'The partition type, there are: H-by hour, D-by day, W-by week, M-by month, O-one-time, R-non-periodical',
@@ -582,15 +596,14 @@ CREATE TABLE `storage_hive`
`status` int(11) DEFAULT '0' COMMENT 'status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
`opt_log` varchar(5000) DEFAULT NULL COMMENT 'Background operation log',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 21
DEFAULT CHARSET = utf8mb4 COMMENT ='Data is stored in Hive configuration table';
-- ----------------------------
@@ -599,13 +612,13 @@ CREATE TABLE `storage_hive`
DROP TABLE IF EXISTS `storage_hive_field`;
CREATE TABLE `storage_hive_field`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
- `source_field_name` varchar(20) DEFAULT NULL COMMENT 'source field name',
- `source_field_type` varchar(20) DEFAULT NULL COMMENT 'source field type',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
- `field_comment` varchar(2000) DEFAULT NULL COMMENT 'Field description',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
+ `source_field_name` varchar(20) NOT NULL COMMENT 'source field name',
+ `source_field_type` varchar(20) NOT NULL COMMENT 'source field type',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
+ `field_comment` varchar(2000) DEFAULT '' COMMENT 'Field description',
`is_required` tinyint(1) DEFAULT NULL COMMENT 'Is it required, 0: not necessary, 1: required',
`bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
`bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
@@ -616,7 +629,6 @@ CREATE TABLE `storage_hive_field`
PRIMARY KEY (`id`),
KEY `index_storage_id` (`storage_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 34
DEFAULT CHARSET = utf8mb4 COMMENT ='Data stored in Hive field';
-- ----------------------------
@@ -625,15 +637,15 @@ CREATE TABLE `storage_hive_field`
DROP TABLE IF EXISTS `task`;
CREATE TABLE `task`
(
- `id` bigint(20) NOT NULL,
- `taskflow_id` bigint(20) DEFAULT NULL COMMENT 'Owning task flow id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_id` bigint(20) NOT NULL COMMENT 'Owning task flow id',
`task_def_id` bigint(20) DEFAULT NULL COMMENT 'task definition id',
- `task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
+ `task_name` varchar(255) NOT NULL COMMENT 'task name',
`status` varchar(255) DEFAULT NULL COMMENT 'task status',
`post_param` varchar(255) DEFAULT NULL COMMENT 'Task parameters',
`resultmsg` varchar(1000) DEFAULT NULL COMMENT 'Execution result log',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(0) DEFAULT NULL COMMENT 'last modified person',
PRIMARY KEY (`id`)
@@ -646,13 +658,13 @@ CREATE TABLE `task`
DROP TABLE IF EXISTS `task_def`;
CREATE TABLE `task_def`
(
- `id` bigint(20) NOT NULL,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Task flow definition id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Task flow definition id',
`parent_id` bigint(20) DEFAULT NULL COMMENT 'parent task id',
`implclass` varchar(255) DEFAULT NULL COMMENT 'task processing flow class',
`task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` datetime DEFAULT NULL COMMENT 'last modified person',
`delivery_id` bigint(20) DEFAULT NULL COMMENT 'Task push method',
@@ -666,11 +678,11 @@ CREATE TABLE `task_def`
DROP TABLE IF EXISTS `taskflow`;
CREATE TABLE `taskflow`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Taskflow definition id',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Taskflow definition id',
`status` varchar(255) DEFAULT NULL COMMENT 'status',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
+ `create_time` datetime DEFAULT NULL COMMENT 'Create time',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(255) DEFAULT NULL COMMENT 'last modified person',
`event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
@@ -684,11 +696,11 @@ CREATE TABLE `taskflow`
DROP TABLE IF EXISTS `taskflow_def`;
CREATE TABLE `taskflow_def`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) DEFAULT NULL COMMENT 'Workflow definition name',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) NOT NULL COMMENT 'Workflow definition name',
`descrip` varchar(255) DEFAULT NULL COMMENT 'Workflow function description',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`isValid` int(11) DEFAULT NULL COMMENT 'logical deletion',
`trigger_event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
PRIMARY KEY (`id`)
@@ -706,14 +718,13 @@ CREATE TABLE `user`
`password` varchar(64) NOT NULL COMMENT 'password md5',
`account_type` int(11) NOT NULL DEFAULT '1' COMMENT 'account type, 0-manager 1-normal',
`due_date` datetime DEFAULT NULL COMMENT 'due date for account',
- `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
`create_by` varchar(255) NOT NULL COMMENT 'create by sb.',
`update_by` varchar(255) DEFAULT NULL COMMENT 'update by sb.',
PRIMARY KEY (`id`),
UNIQUE KEY `user_name_uindex` (`name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 25
DEFAULT CHARSET = utf8mb4 COMMENT ='User table';
-- create default admin user, username is 'admin', password is 'inlong'
@@ -754,13 +765,12 @@ CREATE TABLE `wf_approver`
`approvers` varchar(1024) NOT NULL COMMENT 'Approvers, separated by commas',
`creator` varchar(64) NOT NULL COMMENT 'creator',
`modifier` varchar(64) NOT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
`is_deleted` int(11) DEFAULT '0' COMMENT 'Whether to delete, 0 is not deleted, if greater than 0, delete',
PRIMARY KEY (`id`),
KEY `process_name_task_name_index` (`process_name`, `task_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 5
DEFAULT CHARSET = utf8mb4 COMMENT ='Workflow approver table';
-- create default approver for new consumption and new business
@@ -797,7 +807,6 @@ CREATE TABLE `wf_event_log`
`exception` text COMMENT 'Exception information',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 5170
DEFAULT CHARSET = utf8mb4 COMMENT ='Workflow event log table';
-- ----------------------------
@@ -821,7 +830,6 @@ CREATE TABLE `wf_process_instance`
`hidden` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Is it hidden',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 197
DEFAULT CHARSET = utf8mb4 COMMENT ='Process instance';
-- ----------------------------
@@ -848,7 +856,6 @@ CREATE TABLE `wf_task_instance`
`ext` text COMMENT 'Extended information-json',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 704
DEFAULT CHARSET = utf8mb4 COMMENT ='Task instance';
-- ----------------------------
@@ -859,21 +866,20 @@ CREATE TABLE `cluster_set`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
- `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'ClusterSet Introduction',
- `middleware_type` varchar(10) DEFAULT 'Pulsar' COMMENT 'The middleware type of data storage, high throughput: Pulsar',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
+ `cn_name` varchar(256) COMMENT 'Chinese display name',
+ `description` varchar(256) COMMENT 'ClusterSet Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `in_charges` varchar(512) COMMENT 'Name of responsible person, separated by commas',
+ `followers` varchar(512) COMMENT 'List of names of business followers, separated by commas',
`status` int(11) DEFAULT '21' COMMENT 'ClusterSet status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cluster_set` (`set_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='ClusterSet table';
-- ----------------------------
@@ -888,7 +894,6 @@ CREATE TABLE `cluster_set_inlongid`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cluster_set_inlongid` (`set_name`, `inlong_group_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='InlongId table';
-- ----------------------------
@@ -904,7 +909,6 @@ CREATE TABLE `cache_cluster`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cache_cluster` (`cluster_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='CacheCluster table';
-- ----------------------------
@@ -916,9 +920,9 @@ CREATE TABLE `cache_cluster_ext`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`cluster_name` varchar(128) NOT NULL COMMENT 'CacheCluster name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_cache_cluster` (`cluster_name`)
) ENGINE = InnoDB
@@ -937,7 +941,6 @@ CREATE TABLE `cache_topic`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cache_topic` (`topic_name`, `set_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='CacheTopic table';
-- ----------------------------
@@ -953,7 +956,6 @@ CREATE TABLE `proxy_cluster`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_proxy_cluster` (`cluster_name`, `set_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='ProxyCluster table';
-- ----------------------------
@@ -997,9 +999,9 @@ CREATE TABLE `flume_source_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSource name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_flume_source_ext` (`parent_name`)
) ENGINE = InnoDB
@@ -1030,9 +1032,9 @@ CREATE TABLE `flume_channel_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeChannel name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_flume_channel_ext` (`parent_name`)
) ENGINE = InnoDB
@@ -1064,9 +1066,9 @@ CREATE TABLE `flume_sink_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSink name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_flume_sink_ext` (`parent_name`)
) ENGINE = InnoDB
diff --git a/inlong-manager/doc/sql/apache_inlong_manager.sql b/inlong-manager/doc/sql/apache_inlong_manager.sql
index e09231e..6df8bdc 100644
--- a/inlong-manager/doc/sql/apache_inlong_manager.sql
+++ b/inlong-manager/doc/sql/apache_inlong_manager.sql
@@ -33,7 +33,7 @@ CREATE TABLE `agent_heartbeat_log`
`ip` varchar(64) NOT NULL COMMENT 'agent host ip',
`version` varchar(128) DEFAULT NULL,
`heartbeat_msg` text DEFAULT NULL COMMENT 'massage in heartbeat request',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`ip`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='Agent heartbeat information table';
@@ -65,7 +65,7 @@ CREATE TABLE `agent_sys_conf`
`stat_interval_sec` int(11) NOT NULL DEFAULT '60' COMMENT 'Statistical message sending frequency',
`conf_refresh_interval_secs` int(11) NOT NULL DEFAULT '300' COMMENT 'The frequency at which the Agent regularly pulls the configuration from the InLongManager',
`flow_size` int(11) DEFAULT '1048576000',
- `bufferSize` int(11) DEFAULT NULL COMMENT 'bufferSize, default 1048576',
+ `bufferSize` int(11) DEFAULT '1048576' COMMENT 'bufferSize, default 1048576',
`compress` tinyint(2) DEFAULT NULL COMMENT 'Whether to compress',
`event_check_interval` int(11) DEFAULT NULL COMMENT 'File scanning period',
`is_calMD5` tinyint(2) DEFAULT NULL COMMENT 'Do you want to calculate the cumulative md5 of read characters',
@@ -83,30 +83,53 @@ CREATE TABLE `business`
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
`name` varchar(128) DEFAULT '' COMMENT 'Business name, English, numbers and underscore',
`cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'Business Introduction',
- `middleware_type` varchar(10) DEFAULT 'Tube' COMMENT 'The middleware type of data storage, high throughput: Tube',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, for Tube, its Topic',
- `daily_records` int(11) DEFAULT NULL COMMENT 'Number of access records per day, unit: 10,000 records per day',
- `daily_storage` int(11) DEFAULT NULL COMMENT 'Access size by day, unit: GB per day',
- `peak_records` int(11) DEFAULT NULL COMMENT 'Access peak per second, unit: records per second',
- `max_length` int(11) DEFAULT NULL COMMENT 'The maximum length of a single piece of data, unit: Byte',
+ `description` varchar(256) DEFAULT '' COMMENT 'Business Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `mq_resource_obj` varchar(128) NOT NULL COMMENT 'MQ resource object, for Tube, its Topic, for Pulsar, its Namespace',
+ `daily_records` int(11) DEFAULT '10' COMMENT 'Number of access records per day, unit: 10,000 records per day',
+ `daily_storage` int(11) DEFAULT '10' COMMENT 'Access size by day, unit: GB per day',
+ `peak_records` int(11) DEFAULT '1000' COMMENT 'Access peak per second, unit: records per second',
+ `max_length` int(11) DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte',
`schema_name` varchar(128) DEFAULT NULL COMMENT 'Data type, associated data_schema table',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
`followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
`status` int(11) DEFAULT '21' COMMENT 'Business status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_business` (`inlong_group_id`, `is_deleted`, `modify_time`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='Business table';
-- ----------------------------
+-- Table structure for business_pulsar
+-- ----------------------------
+DROP TABLE IF EXISTS `business_pulsar`;
+CREATE TABLE `business_pulsar`
+(
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
+ `ensemble` int(3) DEFAULT '3' COMMENT 'The writable nodes number of ledger',
+ `write_quorum` int(3) DEFAULT '3' COMMENT 'The copies number of ledger',
+ `ack_quorum` int(3) DEFAULT '2' COMMENT 'The number of requested acks',
+ `retention_time` int(11) DEFAULT '72' COMMENT 'Message storage time',
+ `retention_time_unit` char(20) DEFAULT 'hours' COMMENT 'The unit of the message storage time',
+ `ttl` int(11) DEFAULT '24' COMMENT 'Message time-to-live duration',
+ `ttl_unit` varchar(20) DEFAULT 'hours' COMMENT 'The unit of time-to-live duration',
+ `retention_size` int(11) DEFAULT '-1' COMMENT 'Message size',
+ `retention_size_unit` varchar(20) DEFAULT 'MB' COMMENT 'The unit of message size',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ PRIMARY KEY (`id`)
+) ENGINE = InnoDB
+ DEFAULT CHARSET = utf8mb4 COMMENT ='Pulsar info table';
+
+-- ----------------------------
-- Table structure for business_ext
-- ----------------------------
DROP TABLE IF EXISTS `business_ext`;
@@ -117,7 +140,7 @@ CREATE TABLE `business_ext`
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
`key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_group_id` (`inlong_group_id`)
) ENGINE = InnoDB
@@ -131,19 +154,19 @@ CREATE TABLE `cluster_info`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`name` varchar(128) NOT NULL COMMENT 'cluster name',
- `type` varchar(32) DEFAULT NULL COMMENT 'Cluster type, including TUBE, ZOOKEEPER, etc.',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Cluster IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Cluster port number',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `type` varchar(32) NOT NULL COMMENT 'Cluster type, including TUBE, PULSAR, etc.',
+ `ip` varchar(64) NOT NULL COMMENT 'Cluster IP address',
+ `port` int(11) NOT NULL COMMENT 'Cluster port',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
`url` varchar(256) DEFAULT NULL COMMENT 'Cluster URL address',
`is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
`ext_props` json DEFAULT NULL COMMENT 'extended properties',
`status` int(11) DEFAULT '1' COMMENT 'cluster status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='Cluster Information Table';
@@ -154,35 +177,35 @@ CREATE TABLE `cluster_info`
DROP TABLE IF EXISTS `common_db_server`;
CREATE TABLE `common_db_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
- `db_type` varchar(128) DEFAULT NULL COMMENT 'DB type, such as MySQL, Oracle',
- `db_server_ip` varchar(64) DEFAULT NULL COMMENT 'DB Server IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `username` varchar(64) DEFAULT NULL COMMENT 'Username',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
- `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
- `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
- `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
- `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
- `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
- `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
- `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `connection_name` varchar(128) NOT NULL COMMENT 'The name of the database connection',
+ `db_type` varchar(128) DEFAULT 'MySQL' COMMENT 'DB type, such as MySQL, Oracle',
+ `db_server_ip` varchar(64) NOT NULL COMMENT 'DB Server IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
+ `username` varchar(64) NOT NULL COMMENT 'Username',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
+ `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
+ `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
+ `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
+ `in_charges` varchar(512) NOT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
+ `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
+ `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
+ `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
+ `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- DEFAULT CHARSET = utf8mb4 COMMENT ='public DB data source';
+ DEFAULT CHARSET = utf8mb4 COMMENT ='Common Database Server Table';
-- ----------------------------
-- Table structure for common_file_server
@@ -190,25 +213,25 @@ CREATE TABLE `common_db_server`
DROP TABLE IF EXISTS `common_file_server`;
CREATE TABLE `common_file_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Data source IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
- `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
- `username` varchar(64) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy, LoadProxy',
+ `ip` varchar(64) NOT NULL COMMENT 'Data source IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
+ `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
+ `username` varchar(64) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- DEFAULT CHARSET = utf8mb4 COMMENT ='public file data source';
+ DEFAULT CHARSET = utf8mb4 COMMENT ='Common File Server Table';
-- ----------------------------
-- Table structure for consumption
@@ -219,22 +242,20 @@ CREATE TABLE `consumption`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`consumer_group_name` varchar(255) NOT NULL COMMENT 'consumer group name',
`consumer_group_id` varchar(255) NOT NULL COMMENT 'Consumer group ID',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Person in charge of consumption',
- `inlong_group_id` varchar(255) DEFAULT NULL COMMENT 'Business group id',
- `middleware_type` varchar(64) DEFAULT NULL COMMENT 'The middleware type of data storage, high throughput: Tube',
- `topic` varchar(255) DEFAULT NULL COMMENT 'Consumption topic',
- `filter_enabled` int(2) DEFAULT NULL COMMENT 'Whether to filter',
- `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Data stream ID for consumption',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Person in charge of consumption',
+ `inlong_group_id` varchar(255) NOT NULL COMMENT 'Business group id',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `topic` varchar(255) NOT NULL COMMENT 'Consumption topic',
+ `filter_enabled` int(2) DEFAULT '0' COMMENT 'Whether to filter, default 0, not filter consume',
+ `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Data stream ID for consumption, if filter_enable is 1, it cannot empty',
`status` int(11) NOT NULL COMMENT 'Status: draft, pending approval, approval rejected, approval passed',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator',
+ `creator` varchar(64) NOT NULL COMMENT 'creator',
`modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Modify time',
`is_deleted` int(2) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- PRIMARY KEY (`id`),
- UNIQUE KEY `consumer_group_id_is_deleted_uindex` (`consumer_group_id`, `is_deleted`)
+ PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 6
DEFAULT CHARSET = utf8mb4 COMMENT ='Data consumption configuration table';
-- ----------------------------
@@ -243,22 +264,22 @@ CREATE TABLE `consumption`
DROP TABLE IF EXISTS `data_proxy_cluster`;
CREATE TABLE `data_proxy_cluster`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `name` varchar(128) DEFAULT NULL COMMENT 'cluster name',
- `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
- `address` varchar(128) DEFAULT NULL COMMENT 'cluster address',
- `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
- `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
- `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `ext_props` json DEFAULT NULL COMMENT 'Extended properties',
- `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'Create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `name` varchar(128) NOT NULL COMMENT 'cluster name',
+ `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
+ `address` varchar(128) NOT NULL COMMENT 'cluster address',
+ `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
+ `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
+ `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
+ `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `ext_props` json DEFAULT NULL COMMENT 'Extended properties',
+ `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='DataProxy cluster table';
@@ -278,7 +299,6 @@ CREATE TABLE `data_schema`
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 10
DEFAULT CHARSET = utf8mb4 COMMENT ='Data format table';
-- create default data schema
@@ -297,7 +317,7 @@ CREATE TABLE `data_source_cmd_config`
`specified_data_time` varchar(64) NOT NULL,
`bSend` tinyint(1) NOT NULL,
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Last update time ',
- `create_time` timestamp NULL DEFAULT NULL,
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`result_info` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `index_1` (`task_id`, `bSend`, `specified_data_time`)
@@ -313,9 +333,9 @@ CREATE TABLE `data_stream`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id, non-deleted globally unique',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
- `name` varchar(64) DEFAULT NULL COMMENT 'The name of the data stream page display, can be Chinese',
+ `name` varchar(64) NOT NULL COMMENT 'The name of the data stream page display, can be Chinese',
`description` varchar(256) DEFAULT '' COMMENT 'Introduction to data stream',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id',
+ `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id, Pulsar is Topic',
`data_source_type` varchar(32) DEFAULT 'FILE' COMMENT 'Data source type, including: FILE, DB, Auto-Push (DATA_PROXY_SDK, HTTP)',
`storage_period` int(11) DEFAULT '1' COMMENT 'The storage period of data in MQ, unit: day',
`data_type` varchar(20) DEFAULT 'TEXT' COMMENT 'Data type, there are: TEXT, KEY-VALUE, PB, BON, TEXT and BON should be treated differently',
@@ -326,15 +346,14 @@ CREATE TABLE `data_stream`
`status` int(11) DEFAULT '0' COMMENT 'Data stream status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_data_stream` (`inlong_stream_id`, `inlong_group_id`, `is_deleted`, `modify_time`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 28
DEFAULT CHARSET = utf8mb4 COMMENT ='Data stream table';
-- ----------------------------
@@ -349,7 +368,7 @@ CREATE TABLE `data_stream_ext`
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
`key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_stream_id` (`inlong_stream_id`)
) ENGINE = InnoDB
@@ -365,10 +384,10 @@ CREATE TABLE `data_stream_field`
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
`is_predefined_field` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a predefined field, 0: no, 1: yes',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
`field_value` varchar(128) DEFAULT NULL COMMENT 'Field value, required if it is a predefined field',
`pre_expression` varchar(256) DEFAULT NULL COMMENT 'Pre-defined field value expression',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
`field_comment` varchar(50) DEFAULT NULL COMMENT 'Field description',
`rank_num` smallint(6) DEFAULT '0' COMMENT 'Field order (front-end display field order)',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
@@ -379,7 +398,6 @@ CREATE TABLE `data_stream_field`
PRIMARY KEY (`id`),
KEY `index_stream_id` (`inlong_stream_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 39
DEFAULT CHARSET = utf8mb4 COMMENT ='File/DB data source field table';
-- ----------------------------
@@ -405,7 +423,6 @@ CREATE TABLE `operation_log`
`err_msg` text COMMENT 'Error message',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 289
DEFAULT CHARSET = utf8mb4;
-- ----------------------------
@@ -415,8 +432,8 @@ DROP TABLE IF EXISTS `role`;
CREATE TABLE `role`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
- `role_code` varchar(100) NOT NULL COMMENT 'role code',
- `role_name` varchar(255) NOT NULL COMMENT 'role Chinese name',
+ `role_code` varchar(100) NOT NULL COMMENT 'Role code',
+ `role_name` varchar(255) NOT NULL COMMENT 'Role Chinese name',
`create_time` datetime NOT NULL,
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_by` varchar(255) NOT NULL,
@@ -439,10 +456,10 @@ CREATE TABLE `source_db_basic`
`inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
`sync_type` tinyint(1) DEFAULT '0' COMMENT 'Data synchronization type, 0: FULL, full amount, 1: INCREMENTAL, incremental',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
@@ -457,7 +474,7 @@ CREATE TABLE `source_db_detail`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
`db_name` varchar(128) DEFAULT NULL COMMENT 'database name',
`transfer_ip` varchar(64) DEFAULT NULL COMMENT 'Transfer IP',
`connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
@@ -468,11 +485,11 @@ CREATE TABLE `source_db_detail`
`status` int(11) DEFAULT '0' COMMENT 'Data source status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save unsubmitted and unapproved intermediate data after modification',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='DB data source details table';
@@ -494,14 +511,13 @@ CREATE TABLE `source_file_basic`
`upload_max_size` int(4) DEFAULT '120' COMMENT 'Upload maximum size',
`need_compress` tinyint(1) DEFAULT '0' COMMENT 'Whether need compress',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Delete switch',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator',
`modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'temp view',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 9
DEFAULT CHARSET = utf8mb4 COMMENT ='basic configuration of file data source';
-- ----------------------------
@@ -514,25 +530,24 @@ CREATE TABLE `source_file_detail`
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
`access_type` varchar(20) DEFAULT 'Agent' COMMENT 'Collection type, there are Agent, DataProxy client, LoadProxy, the file can only be Agent temporarily',
- `server_name` varchar(64) DEFAULT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
- `ip` varchar(128) DEFAULT NULL COMMENT 'Data source IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Data source port number',
+ `server_name` varchar(64) NOT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
+ `ip` varchar(128) NOT NULL COMMENT 'Data source IP address',
+ `port` int(11) NOT NULL COMMENT 'Data source port number',
`is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
`issue_type` varchar(10) DEFAULT 'SSH' COMMENT 'Issuing method, there are SSH, TCS',
- `username` varchar(32) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `file_path` varchar(256) DEFAULT NULL COMMENT 'File path, supports regular matching',
+ `username` varchar(32) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `file_path` varchar(256) NOT NULL COMMENT 'File path, supports regular matching',
`status` int(11) DEFAULT '0' COMMENT 'Data source status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
`modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 3
DEFAULT CHARSET = utf8mb4 COMMENT ='Detailed table of file data source';
-- ----------------------------
@@ -547,11 +562,10 @@ CREATE TABLE `storage_ext`
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
`key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_storage_id` (`storage_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 11
DEFAULT CHARSET = utf8mb4 COMMENT ='Data storage extension table';
-- ----------------------------
@@ -564,11 +578,11 @@ CREATE TABLE `storage_hive`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `jdbc_url` varchar(255) DEFAULT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
- `username` varchar(128) DEFAULT NULL COMMENT 'Username',
- `password` varchar(255) DEFAULT NULL COMMENT 'User password',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `table_name` varchar(128) DEFAULT NULL COMMENT 'Target data table name',
+ `jdbc_url` varchar(255) NOT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
+ `username` varchar(128) NOT NULL COMMENT 'Username',
+ `password` varchar(255) NOT NULL COMMENT 'User password',
+ `db_name` varchar(128) NOT NULL COMMENT 'Target database name',
+ `table_name` varchar(128) NOT NULL COMMENT 'Target data table name',
`primary_partition` varchar(255) DEFAULT 'dt' COMMENT 'primary partition field',
`secondary_partition` varchar(256) DEFAULT NULL COMMENT 'secondary partition field',
`partition_type` varchar(10) DEFAULT NULL COMMENT 'The partition type, there are: H-by hour, D-by day, W-by week, M-by month, O-one-time, R-non-periodical',
@@ -582,15 +596,14 @@ CREATE TABLE `storage_hive`
`status` int(11) DEFAULT '0' COMMENT 'status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` json DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
`opt_log` varchar(5000) DEFAULT NULL COMMENT 'Background operation log',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 21
DEFAULT CHARSET = utf8mb4 COMMENT ='Data is stored in Hive configuration table';
-- ----------------------------
@@ -599,13 +612,13 @@ CREATE TABLE `storage_hive`
DROP TABLE IF EXISTS `storage_hive_field`;
CREATE TABLE `storage_hive_field`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
- `source_field_name` varchar(20) DEFAULT NULL COMMENT 'source field name',
- `source_field_type` varchar(20) DEFAULT NULL COMMENT 'source field type',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
- `field_comment` varchar(2000) DEFAULT NULL COMMENT 'Field description',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
+ `source_field_name` varchar(20) NOT NULL COMMENT 'source field name',
+ `source_field_type` varchar(20) NOT NULL COMMENT 'source field type',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
+ `field_comment` varchar(2000) DEFAULT '' COMMENT 'Field description',
`is_required` tinyint(1) DEFAULT NULL COMMENT 'Is it required, 0: not necessary, 1: required',
`bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
`bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
@@ -616,7 +629,6 @@ CREATE TABLE `storage_hive_field`
PRIMARY KEY (`id`),
KEY `index_storage_id` (`storage_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 34
DEFAULT CHARSET = utf8mb4 COMMENT ='Data stored in Hive field';
-- ----------------------------
@@ -625,15 +637,15 @@ CREATE TABLE `storage_hive_field`
DROP TABLE IF EXISTS `task`;
CREATE TABLE `task`
(
- `id` bigint(20) NOT NULL,
- `taskflow_id` bigint(20) DEFAULT NULL COMMENT 'Owning task flow id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_id` bigint(20) NOT NULL COMMENT 'Owning task flow id',
`task_def_id` bigint(20) DEFAULT NULL COMMENT 'task definition id',
- `task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
+ `task_name` varchar(255) NOT NULL COMMENT 'task name',
`status` varchar(255) DEFAULT NULL COMMENT 'task status',
`post_param` varchar(255) DEFAULT NULL COMMENT 'Task parameters',
`resultmsg` varchar(1000) DEFAULT NULL COMMENT 'Execution result log',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(0) DEFAULT NULL COMMENT 'last modified person',
PRIMARY KEY (`id`)
@@ -646,13 +658,13 @@ CREATE TABLE `task`
DROP TABLE IF EXISTS `task_def`;
CREATE TABLE `task_def`
(
- `id` bigint(20) NOT NULL,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Task flow definition id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Task flow definition id',
`parent_id` bigint(20) DEFAULT NULL COMMENT 'parent task id',
`implclass` varchar(255) DEFAULT NULL COMMENT 'task processing flow class',
`task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` datetime DEFAULT NULL COMMENT 'last modified person',
`delivery_id` bigint(20) DEFAULT NULL COMMENT 'Task push method',
@@ -666,11 +678,11 @@ CREATE TABLE `task_def`
DROP TABLE IF EXISTS `taskflow`;
CREATE TABLE `taskflow`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Taskflow definition id',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Taskflow definition id',
`status` varchar(255) DEFAULT NULL COMMENT 'status',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
+ `create_time` datetime DEFAULT NULL COMMENT 'Create time',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(255) DEFAULT NULL COMMENT 'last modified person',
`event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
@@ -684,11 +696,11 @@ CREATE TABLE `taskflow`
DROP TABLE IF EXISTS `taskflow_def`;
CREATE TABLE `taskflow_def`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) DEFAULT NULL COMMENT 'Workflow definition name',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) NOT NULL COMMENT 'Workflow definition name',
`descrip` varchar(255) DEFAULT NULL COMMENT 'Workflow function description',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`isValid` int(11) DEFAULT NULL COMMENT 'logical deletion',
`trigger_event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
PRIMARY KEY (`id`)
@@ -706,14 +718,13 @@ CREATE TABLE `user`
`password` varchar(64) NOT NULL COMMENT 'password md5',
`account_type` int(11) NOT NULL DEFAULT '1' COMMENT 'account type, 0-manager 1-normal',
`due_date` datetime DEFAULT NULL COMMENT 'due date for account',
- `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
`create_by` varchar(255) NOT NULL COMMENT 'create by sb.',
`update_by` varchar(255) DEFAULT NULL COMMENT 'update by sb.',
PRIMARY KEY (`id`),
UNIQUE KEY `user_name_uindex` (`name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 25
DEFAULT CHARSET = utf8mb4 COMMENT ='User table';
-- create default admin user, username is 'admin', password is 'inlong'
@@ -754,13 +765,12 @@ CREATE TABLE `wf_approver`
`approvers` varchar(1024) NOT NULL COMMENT 'Approvers, separated by commas',
`creator` varchar(64) NOT NULL COMMENT 'creator',
`modifier` varchar(64) NOT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
`is_deleted` int(11) DEFAULT '0' COMMENT 'Whether to delete, 0 is not deleted, if greater than 0, delete',
PRIMARY KEY (`id`),
KEY `process_name_task_name_index` (`process_name`, `task_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 5
DEFAULT CHARSET = utf8mb4 COMMENT ='Workflow approver table';
-- create default approver for new consumption and new business
@@ -797,7 +807,6 @@ CREATE TABLE `wf_event_log`
`exception` text COMMENT 'Exception information',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 5170
DEFAULT CHARSET = utf8mb4 COMMENT ='Workflow event log table';
-- ----------------------------
@@ -821,7 +830,6 @@ CREATE TABLE `wf_process_instance`
`hidden` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Is it hidden',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 197
DEFAULT CHARSET = utf8mb4 COMMENT ='Process instance';
-- ----------------------------
@@ -848,7 +856,6 @@ CREATE TABLE `wf_task_instance`
`ext` text COMMENT 'Extended information-json',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 704
DEFAULT CHARSET = utf8mb4 COMMENT ='Task instance';
-- ----------------------------
@@ -859,21 +866,20 @@ CREATE TABLE `cluster_set`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
- `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'ClusterSet Introduction',
- `middleware_type` varchar(10) DEFAULT 'Pulsar' COMMENT 'The middleware type of data storage, high throughput: Pulsar',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
+ `cn_name` varchar(256) COMMENT 'Chinese display name',
+ `description` varchar(256) COMMENT 'ClusterSet Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `in_charges` varchar(512) COMMENT 'Name of responsible person, separated by commas',
+ `followers` varchar(512) COMMENT 'List of names of business followers, separated by commas',
`status` int(11) DEFAULT '21' COMMENT 'ClusterSet status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cluster_set` (`set_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='ClusterSet table';
-- ----------------------------
@@ -888,7 +894,6 @@ CREATE TABLE `cluster_set_inlongid`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cluster_set_inlongid` (`set_name`, `inlong_group_id`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='InlongId table';
-- ----------------------------
@@ -904,7 +909,6 @@ CREATE TABLE `cache_cluster`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cache_cluster` (`cluster_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='CacheCluster table';
-- ----------------------------
@@ -916,9 +920,9 @@ CREATE TABLE `cache_cluster_ext`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`cluster_name` varchar(128) NOT NULL COMMENT 'CacheCluster name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_cache_cluster` (`cluster_name`)
) ENGINE = InnoDB
@@ -937,7 +941,6 @@ CREATE TABLE `cache_topic`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_cache_topic` (`topic_name`, `set_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='CacheTopic table';
-- ----------------------------
@@ -953,7 +956,6 @@ CREATE TABLE `proxy_cluster`
PRIMARY KEY (`id`),
UNIQUE KEY `unique_proxy_cluster` (`cluster_name`, `set_name`)
) ENGINE = InnoDB
- AUTO_INCREMENT = 16
DEFAULT CHARSET = utf8mb4 COMMENT ='ProxyCluster table';
-- ----------------------------
@@ -997,9 +999,9 @@ CREATE TABLE `flume_source_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSource name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_flume_source_ext` (`parent_name`)
) ENGINE = InnoDB
@@ -1030,9 +1032,9 @@ CREATE TABLE `flume_channel_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeChannel name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_flume_channel_ext` (`parent_name`)
) ENGINE = InnoDB
@@ -1064,9 +1066,9 @@ CREATE TABLE `flume_sink_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSink name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`),
KEY `index_flume_sink_ext` (`parent_name`)
) ENGINE = InnoDB
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java
index 75a3305..78185bd 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java
@@ -43,4 +43,10 @@ public class ClusterBean {
@Value("${sort.appName}")
private String appName;
+ @Value("${pulsar.adminUrl}")
+ private String pulsarAdminUrl;
+
+ @Value("${pulsar.serviceUrl}")
+ private String pulsarServiceUrl;
+
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/TryBean.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ReTryConfigBean.java
similarity index 97%
rename from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/TryBean.java
rename to inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ReTryConfigBean.java
index 8bcee60..c9fa1ab 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/TryBean.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ReTryConfigBean.java
@@ -22,7 +22,7 @@ import org.springframework.stereotype.Component;
@Data
@Component
-public class TryBean {
+public class ReTryConfigBean {
/**
* Maximum number of attempts
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/BizConstant.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/BizConstant.java
index a18827e..a5589b0 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/BizConstant.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/BizConstant.java
@@ -22,19 +22,21 @@ package org.apache.inlong.manager.common.enums;
*/
public class BizConstant {
- public static final String STORAGE_TYPE_HIVE = "HIVE";
+ public static final String STORAGE_HIVE = "HIVE";
- public static final String DATA_SOURCE_TYPE_DB = "DB";
+ public static final String DATA_SOURCE_DB = "DB";
- public static final String DATA_SOURCE_TYPE_FILE = "FILE";
+ public static final String DATA_SOURCE_FILE = "FILE";
- public static final String DATA_SOURCE_TYPE_AUTO_PUSH = "AUTO_PUSH";
+ public static final String DATA_SOURCE_AUTO_PUSH = "AUTO_PUSH";
public static final String DATA_TYPE_TEXT = "TEXT";
public static final String DATA_TYPE_KEY_VALUE = "KEY-VALUE";
- public static final String MIDDLEWARE_TYPE_TUBE = "TUBE";
+ public static final String MIDDLEWARE_TUBE = "TUBE";
+
+ public static final String MIDDLEWARE_PULSAR = "PULSAR";
public static final String SCHEMA_M0_DAY = "m0_day";
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java
index fbf683f..5824bb1 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java
@@ -29,6 +29,10 @@ public class BaseException extends RuntimeException {
int code;
String message;
+ public BaseException(String message) {
+ this.message = message;
+ }
+
public BaseException(int code, String message) {
this.code = code;
this.message = message;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BusinessException.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BusinessException.java
index 762ee22..55e0e10 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BusinessException.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BusinessException.java
@@ -31,6 +31,10 @@ public class BusinessException extends BaseException {
super(errorCodeEnum.getCode(), errorCodeEnum.getDefaultMessage());
}
+ public BusinessException(String message) {
+ super(message);
+ }
+
public BusinessException(BizErrorCodeEnum errorCodeEnum, String message) {
super(errorCodeEnum.getCode(), message);
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
index 4cf88bb..b417064 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
@@ -43,4 +43,34 @@ public class BusinessApproveInfo {
@ApiModelProperty(value = "Data type name")
private String schemaName;
+ @ApiModelProperty(value = "The partition num of Pulsar topic, between 1-20")
+ private Integer topicPartitionNum;
+
+ @ApiModelProperty(value = "Ledger's number of writable nodes")
+ private Integer ensemble;
+
+ @ApiModelProperty(value = "Ledger's number of copies")
+ private Integer writeQuorum;
+
+ @ApiModelProperty(value = "Number of responses requested")
+ private Integer ackQuorum;
+
+ @ApiModelProperty(value = "Message storage time")
+ private Integer retentionTime;
+
+ @ApiModelProperty(value = "The unit of the message storage time")
+ private String retentionTimeUnit;
+
+ @ApiModelProperty(value = "Message time-to-live duration")
+ private Integer ttl;
+
+ @ApiModelProperty(value = "The unit of message's time-to-live duration")
+ private String ttlUnit;
+
+ @ApiModelProperty(value = "Message size")
+ private Integer retentionSize;
+
+ @ApiModelProperty(value = "The unit of message size")
+ private String retentionSizeUnit;
+
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessInfo.java
index 712f3ec..db87278 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessInfo.java
@@ -52,12 +52,21 @@ public class BusinessInfo {
@ApiModelProperty(value = "Business description")
private String description;
- @ApiModelProperty(value = "Middleware type of data storage, high throughput: TUBE")
+ @ApiModelProperty(value = "Middleware type, high throughput: TUBE, high consistency: PULSAR")
private String middlewareType;
@ApiModelProperty(value = "MQ resource object, in business, Tube corresponds to Topic")
private String mqResourceObj;
+ @ApiModelProperty(value = "Tube master URL")
+ private String tubeMaster;
+
+ @ApiModelProperty(value = "Pulsar admin URL")
+ private String pulsarAdminUrl;
+
+ @ApiModelProperty(value = "Pulsar service URL")
+ private String pulsarServiceUrl;
+
@ApiModelProperty(value = "Data type name")
private String schemaName;
@@ -99,4 +108,8 @@ public class BusinessInfo {
@ApiModelProperty(value = "Business Extension Properties")
private List<BusinessExtInfo> extList;
+
+ @ApiModelProperty(value = "The extension info for MQ")
+ private BusinessMqExtBase mqExtInfo;
+
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessMqExtBase.java
similarity index 60%
copy from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
copy to inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessMqExtBase.java
index 4cf88bb..6f365e6 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessMqExtBase.java
@@ -17,16 +17,24 @@
package org.apache.inlong.manager.common.pojo.business;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
+import org.apache.inlong.manager.common.enums.BizConstant;
/**
- * Business access approval information
+ * Extended business information of different MQs
*/
@Data
-@ApiModel("Business Access Approval Information")
-public class BusinessApproveInfo {
+@ApiModel("Extended business information of different MQs")
+@JsonTypeInfo(use = Id.NAME, visible = true, property = "middlewareType", defaultImpl = BusinessMqExtBase.class)
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = BusinessPulsarInfo.class, name = BizConstant.MIDDLEWARE_PULSAR)
+})
+public class BusinessMqExtBase {
@ApiModelProperty(value = "Primary key")
private Integer id;
@@ -34,13 +42,10 @@ public class BusinessApproveInfo {
@ApiModelProperty(value = "Business group id", required = true)
private String inlongGroupId;
- @ApiModelProperty(value = "Middleware Type")
- private String middlewareType;
-
- @ApiModelProperty(value = "MQ resource object, for Tube, it's Topic")
- private String mqResourceObj;
+ @ApiModelProperty(value = "is deleted? 0: deleted, 1: not deleted")
+ private Integer isDeleted = 0;
- @ApiModelProperty(value = "Data type name")
- private String schemaName;
+ @ApiModelProperty(value = "Middleware type of data storage, high throughput: TUBE")
+ private String middlewareType;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessPulsarInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessPulsarInfo.java
new file mode 100644
index 0000000..8b15b3e
--- /dev/null
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessPulsarInfo.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.common.pojo.business;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import org.apache.inlong.manager.common.enums.BizConstant;
+
+/**
+ * Business access information for Pulsar
+ */
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@ApiModel("Business access information for Pulsar")
+public class BusinessPulsarInfo extends BusinessMqExtBase {
+
+ @ApiModelProperty(value = "Type of middleware")
+ private String middlewareType = BizConstant.MIDDLEWARE_PULSAR;
+
+ @ApiModelProperty(value = "Ledger's number of writable nodes")
+ private Integer ensemble = 3;
+
+ @ApiModelProperty(value = "Ledger's number of copies")
+ private Integer writeQuorum = 3;
+
+ @ApiModelProperty(value = "Number of responses requested")
+ private Integer ackQuorum = 2;
+
+ @ApiModelProperty(value = "Message storage time")
+ private Integer retentionTime = 72;
+
+ @ApiModelProperty(value = "The unit of the message storage time")
+ private String retentionTimeUnit;
+
+ @ApiModelProperty(value = "Message time-to-live duration")
+ private Integer ttl = 24;
+
+ @ApiModelProperty(value = "The unit of message's time-to-live duration")
+ private String ttlUnit;
+
+ @ApiModelProperty(value = "Message size")
+ private Integer retentionSize = -1;
+
+ @ApiModelProperty(value = "The unit of message size")
+ private String retentionSizeUnit;
+
+}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessTopicVO.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessTopicVO.java
index 03ab786..94553a9 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessTopicVO.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessTopicVO.java
@@ -19,7 +19,9 @@ package org.apache.inlong.manager.common.pojo.business;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.util.List;
import lombok.Data;
+import org.apache.inlong.manager.common.pojo.datastream.DataStreamTopicVO;
/**
* Topic View Object of the Business
@@ -31,13 +33,23 @@ public class BusinessTopicVO {
@ApiModelProperty(value = "Business group id", required = true)
private String inlongGroupId;
- @ApiModelProperty(value = "Middleware type of data storage, support high throughput: Tube")
+ @ApiModelProperty(value = "Middleware type, high throughput: TUBE, high consistency: PULSAR")
private String middlewareType;
@ApiModelProperty(value = "Tube topic name")
private String topicName;
- @ApiModelProperty(value = "URL of middleware cluster")
- private String masterUrl;
+ @ApiModelProperty(value = "Topic list, Tube corresponds to business group, there is only 1 topic, "
+ + "Pulsar corresponds to data stream, there are multiple topics")
+ private List<DataStreamTopicVO> dsTopicList;
+
+ @ApiModelProperty(value = "Tube master URL")
+ private String tubeMasterUrl;
+
+ @ApiModelProperty(value = "Pulsar service URL")
+ private String pulsarServiceUrl;
+
+ @ApiModelProperty(value = "Pulsar admin URL")
+ private String pulsarAdminUrl;
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionInfo.java
index b4817a8..2834050 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionInfo.java
@@ -59,7 +59,7 @@ public class ConsumptionInfo {
@NotBlank(message = "business group id cannot be null")
private String inlongGroupId;
- @ApiModelProperty(value = "middleware type for data storage, support Tube, which has high throughput")
+ @ApiModelProperty(value = "Middleware type, high throughput: TUBE, high consistency: PULSAR")
@NotBlank(message = "middlewareType can't be null")
private String middlewareType;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionListVo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionListVo.java
index 66e0894..9ac259c 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionListVo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionListVo.java
@@ -50,7 +50,7 @@ public class ConsumptionListVo {
@ApiModelProperty(value = "Consumption target business group id")
private String inlongGroupId;
- @ApiModelProperty(value = "The middleware type of data storage, high throughput: TUBE")
+ @ApiModelProperty(value = "Middleware type, high throughput: TUBE, high consistency: PULSAR")
private String middlewareType;
@ApiModelProperty(value = "Consumption target TOPIC")
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionQuery.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionQuery.java
index 6a50630..b786125 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionQuery.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/dataconsumption/ConsumptionQuery.java
@@ -46,7 +46,7 @@ public class ConsumptionQuery extends PageRequest {
@ApiModelProperty(value = "Consumption target business group id")
private String inlongGroupId;
- @ApiModelProperty(value = "The middleware type of data storage, high throughput: Tube")
+ @ApiModelProperty(value = "Middleware type, high throughput: TUBE, high consistency: PULSAR")
private String middlewareType;
@ApiModelProperty(value = "Consumption target TOPIC")
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/BaseStorageInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/BaseStorageInfo.java
index 1ebd915..2832839 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/BaseStorageInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/BaseStorageInfo.java
@@ -35,7 +35,7 @@ import org.apache.inlong.manager.common.enums.BizConstant;
@ApiModel("Basic data storage information")
@JsonTypeInfo(use = Id.NAME, visible = true, property = "storageType")
@JsonSubTypes({
- @Type(value = StorageHiveInfo.class, name = BizConstant.STORAGE_TYPE_HIVE)
+ @Type(value = StorageHiveInfo.class, name = BizConstant.STORAGE_HIVE)
})
public class BaseStorageInfo {
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/StorageHiveInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/StorageHiveInfo.java
index 01c4bef..589a262 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/StorageHiveInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastorage/StorageHiveInfo.java
@@ -34,7 +34,7 @@ import org.apache.inlong.manager.common.enums.BizConstant;
@ApiModel(value = "Interaction objects for Hive storage")
public class StorageHiveInfo extends BaseStorageInfo {
- private String storageType = BizConstant.STORAGE_TYPE_HIVE;
+ private String storageType = BizConstant.STORAGE_HIVE;
@ApiModelProperty("Hive JDBC URL")
private String jdbcUrl;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastream/DataStreamTopicVO.java
similarity index 61%
copy from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
copy to inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastream/DataStreamTopicVO.java
index 4cf88bb..2584d1b 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/business/BusinessApproveInfo.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/pojo/datastream/DataStreamTopicVO.java
@@ -15,32 +15,23 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.common.pojo.business;
+package org.apache.inlong.manager.common.pojo.datastream;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
- * Business access approval information
+ * Topic View Object of the Data stream
*/
@Data
-@ApiModel("Business Access Approval Information")
-public class BusinessApproveInfo {
+@ApiModel("Data stream ID and topic interaction object")
+public class DataStreamTopicVO {
- @ApiModelProperty(value = "Primary key")
- private Integer id;
+ @ApiModelProperty(value = "InLong stream ID")
+ private String inlongStreamId;
- @ApiModelProperty(value = "Business group id", required = true)
- private String inlongGroupId;
-
- @ApiModelProperty(value = "Middleware Type")
- private String middlewareType;
-
- @ApiModelProperty(value = "MQ resource object, for Tube, it's Topic")
+ @ApiModelProperty(value = "Message queue type, the data stream id corresponds to the topic of Pulsar one-to-one")
private String mqResourceObj;
- @ApiModelProperty(value = "Data type name")
- private String schemaName;
-
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/BusinessPulsarEntity.java
similarity index 59%
copy from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java
copy to inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/BusinessPulsarEntity.java
index fbf683f..52a5657 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/exceptions/BaseException.java
+++ b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/BusinessPulsarEntity.java
@@ -15,22 +15,26 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.common.exceptions;
+package org.apache.inlong.manager.dao.entity;
+import java.io.Serializable;
import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
@Data
-@NoArgsConstructor
-@EqualsAndHashCode(callSuper = false)
-public class BaseException extends RuntimeException {
+public class BusinessPulsarEntity implements Serializable {
- int code;
- String message;
+ private static final long serialVersionUID = 1L;
+ private Integer id;
+ private String inlongGroupId;
+ private Integer ensemble;
+ private Integer writeQuorum;
+ private Integer ackQuorum;
+ private Integer retentionTime;
+ private String retentionTimeUnit;
+ private Integer ttl;
+ private String ttlUnit;
+ private Integer retentionSize;
+ private String retentionSizeUnit;
+ private Integer isDeleted;
- public BaseException(int code, String message) {
- this.code = code;
- this.message = message;
- }
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/BusinessPulsarEntityMapper.java
similarity index 53%
copy from inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java
copy to inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/BusinessPulsarEntityMapper.java
index 75a3305..8491a39 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/beans/ClusterBean.java
+++ b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/BusinessPulsarEntityMapper.java
@@ -15,32 +15,30 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.common.beans;
+package org.apache.inlong.manager.dao.mapper;
-import lombok.Data;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
+import org.apache.inlong.manager.dao.entity.BusinessPulsarEntity;
+import org.springframework.stereotype.Repository;
-@Data
-@Component
-public class ClusterBean {
+@Repository
+public interface BusinessPulsarEntityMapper {
- @Value("${cluster.tube.master}")
- private String tubeMaster;
+ int deleteByPrimaryKey(Integer id);
- @Value("${cluster.tube.manager}")
- private String tubeManager;
+ int logicDeleteByGroupId(String groupId);
- @Value("${cluster.tube.clusterId}")
- private Integer clusterId;
+ int insert(BusinessPulsarEntity record);
- @Value("${cluster.zk.url}")
- private String zkUrl;
+ int insertSelective(BusinessPulsarEntity record);
- @Value("${cluster.zk.root}")
- private String zkRoot;
+ BusinessPulsarEntity selectByPrimaryKey(Integer id);
- @Value("${sort.appName}")
- private String appName;
+ BusinessPulsarEntity selectByGroupId(String groupId);
-}
+ int updateByPrimaryKeySelective(BusinessPulsarEntity record);
+
+ int updateByPrimaryKey(BusinessPulsarEntity record);
+
+ int updateByIdentifierSelective(BusinessPulsarEntity record);
+
+}
\ No newline at end of file
diff --git a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/DataStreamEntityMapper.java b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/DataStreamEntityMapper.java
index e09f60b..257b085 100644
--- a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/DataStreamEntityMapper.java
+++ b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/DataStreamEntityMapper.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.apache.inlong.manager.common.pojo.datastream.DataStreamInfoToHiveConfig;
import org.apache.inlong.manager.common.pojo.datastream.DataStreamPageRequest;
+import org.apache.inlong.manager.common.pojo.datastream.DataStreamTopicVO;
import org.apache.inlong.manager.dao.entity.DataStreamEntity;
import org.springframework.stereotype.Repository;
@@ -68,10 +69,10 @@ public interface DataStreamEntityMapper {
DataStreamInfoToHiveConfig selectStreamToHiveInfoByIdentifier(@Param("groupId") String groupId,
@Param("streamId") String streamId);
- int updateByIdentifierSelective(DataStreamEntity streamEntity);
-
int selectCountByGroupId(@Param("groupId") String groupId);
+ List<DataStreamTopicVO> selectTopicList(@Param("groupId") String groupId);
+
/**
* Physically delete all data streams of the specified business group id
*
@@ -79,6 +80,8 @@ public interface DataStreamEntityMapper {
*/
int deleteAllByGroupId(@Param("groupId") String groupId);
+ int updateByIdentifierSelective(DataStreamEntity streamEntity);
+
int updateStatusByIdentifier(@Param("groupId") String groupId, @Param("streamId") String streamId,
@Param("status") Integer status, @Param("modifier") String modifier);
diff --git a/inlong-manager/manager-dao/src/main/resources/mappers/BusinessPulsarEntityMapper.xml b/inlong-manager/manager-dao/src/main/resources/mappers/BusinessPulsarEntityMapper.xml
new file mode 100644
index 0000000..f2c4de8
--- /dev/null
+++ b/inlong-manager/manager-dao/src/main/resources/mappers/BusinessPulsarEntityMapper.xml
@@ -0,0 +1,251 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.apache.inlong.manager.dao.mapper.BusinessPulsarEntityMapper">
+ <resultMap id="BaseResultMap" type="org.apache.inlong.manager.dao.entity.BusinessPulsarEntity">
+ <id column="id" jdbcType="INTEGER" property="id"/>
+ <result column="inlong_group_id" jdbcType="VARCHAR" property="inlongGroupId"/>
+ <result column="ensemble" jdbcType="INTEGER" property="ensemble"/>
+ <result column="write_quorum" jdbcType="INTEGER" property="writeQuorum"/>
+ <result column="ack_quorum" jdbcType="INTEGER" property="ackQuorum"/>
+ <result column="retention_time" jdbcType="INTEGER" property="retentionTime"/>
+ <result column="retention_time_unit" jdbcType="CHAR" property="retentionTimeUnit"/>
+ <result column="ttl" jdbcType="INTEGER" property="ttl"/>
+ <result column="ttl_unit" jdbcType="VARCHAR" property="ttlUnit"/>
+ <result column="retention_size" jdbcType="INTEGER" property="retentionSize"/>
+ <result column="retention_size_unit" jdbcType="VARCHAR" property="retentionSizeUnit"/>
+ <result column="is_deleted" jdbcType="INTEGER" property="isDeleted"/>
+ </resultMap>
+ <sql id="Base_Column_List">
+ id, inlong_group_id, ensemble, write_quorum, ack_quorum, retention_time, retention_time_unit,
+ ttl, ttl_unit, retention_size, retention_size_unit, is_deleted
+ </sql>
+ <select id="selectByPrimaryKey" parameterType="java.lang.Integer" resultMap="BaseResultMap">
+ select
+ <include refid="Base_Column_List"/>
+ from business_pulsar
+ where id = #{id,jdbcType=INTEGER}
+ and is_deleted = 0
+ </select>
+ <select id="selectByGroupId" parameterType="java.lang.String" resultMap="BaseResultMap">
+ select
+ <include refid="Base_Column_List"/>
+ from business_pulsar
+ where inlong_group_id = #{inlongGroupId,jdbcType=VARCHAR}
+ and is_deleted = 0
+ </select>
+
+ <delete id="deleteByPrimaryKey" parameterType="java.lang.Integer">
+ update business_pulsar
+ set is_deleted=1
+ where id = #{id,jdbcType=INTEGER}
+ and is_deleted = 0
+ </delete>
+ <update id="logicDeleteByGroupId">
+ update business_pulsar
+ set is_deleted = 1
+ where inlong_group_id = #{inlongGroupId, jdbcType=VARCHAR}
+ and is_deleted = 0
+ </update>
+
+ <insert id="insert" parameterType="org.apache.inlong.manager.dao.entity.BusinessPulsarEntity">
+ insert into business_pulsar (id, inlong_group_id, ensemble,
+ write_quorum, ack_quorum, retention_time,
+ retention_time_unit, ttl, ttl_unit,
+ retention_size, retention_size_unit, is_deleted)
+ values (#{id,jdbcType=INTEGER}, #{inlongGroupId,jdbcType=VARCHAR}, #{ensemble,jdbcType=INTEGER},
+ #{writeQuorum,jdbcType=INTEGER}, #{ackQuorum,jdbcType=INTEGER}, #{retentionTime,jdbcType=INTEGER},
+ #{retentionTimeUnit,jdbcType=CHAR}, #{ttl,jdbcType=INTEGER}, #{ttlUnit,jdbcType=VARCHAR},
+ #{retentionSize,jdbcType=INTEGER}, #{retentionSizeUnit,jdbcType=VARCHAR}, #{isDeleted,jdbcType=INTEGER})
+ </insert>
+ <insert id="insertSelective" parameterType="org.apache.inlong.manager.dao.entity.BusinessPulsarEntity">
+ insert into business_pulsar
+ <trim prefix="(" suffix=")" suffixOverrides=",">
+ <if test="id != null">
+ id,
+ </if>
+ <if test="inlongGroupId != null">
+ inlong_group_id,
+ </if>
+ <if test="ensemble != null">
+ ensemble,
+ </if>
+ <if test="writeQuorum != null">
+ write_quorum,
+ </if>
+ <if test="ackQuorum != null">
+ ack_quorum,
+ </if>
+ <if test="retentionTime != null">
+ retention_time,
+ </if>
+ <if test="retentionTimeUnit != null">
+ retention_time_unit,
+ </if>
+ <if test="ttl != null">
+ ttl,
+ </if>
+ <if test="ttlUnit != null">
+ ttl_unit,
+ </if>
+ <if test="retentionSize != null">
+ retention_size,
+ </if>
+ <if test="retentionSizeUnit != null">
+ retention_size_unit,
+ </if>
+ <if test="isDeleted != null">
+ is_deleted,
+ </if>
+ </trim>
+ <trim prefix="values (" suffix=")" suffixOverrides=",">
+ <if test="id != null">
+ #{id,jdbcType=INTEGER},
+ </if>
+ <if test="inlongGroupId != null">
+ #{inlongGroupId,jdbcType=VARCHAR},
+ </if>
+ <if test="ensemble != null">
+ #{ensemble,jdbcType=INTEGER},
+ </if>
+ <if test="writeQuorum != null">
+ #{writeQuorum,jdbcType=INTEGER},
+ </if>
+ <if test="ackQuorum != null">
+ #{ackQuorum,jdbcType=INTEGER},
+ </if>
+ <if test="retentionTime != null">
+ #{retentionTime,jdbcType=INTEGER},
+ </if>
+ <if test="retentionTimeUnit != null">
+ #{retentionTimeUnit,jdbcType=CHAR},
+ </if>
+ <if test="ttl != null">
+ #{ttl,jdbcType=INTEGER},
+ </if>
+ <if test="ttlUnit != null">
+ #{ttlUnit,jdbcType=VARCHAR},
+ </if>
+ <if test="retentionSize != null">
+ #{retentionSize,jdbcType=INTEGER},
+ </if>
+ <if test="retentionSizeUnit != null">
+ #{retentionSizeUnit,jdbcType=VARCHAR},
+ </if>
+ <if test="isDeleted != null">
+ #{isDeleted,jdbcType=INTEGER},
+ </if>
+ </trim>
+ </insert>
+ <update id="updateByPrimaryKeySelective" parameterType="org.apache.inlong.manager.dao.entity.BusinessPulsarEntity">
+ update business_pulsar
+ <set>
+ <if test="inlongGroupId != null">
+ inlong_group_id = #{inlongGroupId,jdbcType=VARCHAR},
+ </if>
+ <if test="ensemble != null">
+ ensemble = #{ensemble,jdbcType=INTEGER},
+ </if>
+ <if test="writeQuorum != null">
+ write_quorum = #{writeQuorum,jdbcType=INTEGER},
+ </if>
+ <if test="ackQuorum != null">
+ ack_quorum = #{ackQuorum,jdbcType=INTEGER},
+ </if>
+ <if test="retentionTime != null">
+ retention_time = #{retentionTime,jdbcType=INTEGER},
+ </if>
+ <if test="retentionTimeUnit != null">
+ retention_time_unit = #{retentionTimeUnit,jdbcType=CHAR},
+ </if>
+ <if test="ttl != null">
+ ttl = #{ttl,jdbcType=INTEGER},
+ </if>
+ <if test="ttlUnit != null">
+ ttl_unit = #{ttlUnit,jdbcType=VARCHAR},
+ </if>
+ <if test="retentionSize != null">
+ retention_size = #{retentionSize,jdbcType=INTEGER},
+ </if>
+ <if test="retentionSizeUnit != null">
+ retention_size_unit = #{retentionSizeUnit,jdbcType=VARCHAR},
+ </if>
+ <if test="isDeleted != null">
+ is_deleted = #{isDeleted,jdbcType=INTEGER},
+ </if>
+ </set>
+ where id = #{id,jdbcType=INTEGER} and is_deleted=0
+ </update>
+ <update id="updateByPrimaryKey" parameterType="org.apache.inlong.manager.dao.entity.BusinessPulsarEntity">
+ update business_pulsar
+ set inlong_group_id = #{inlongGroupId,jdbcType=VARCHAR},
+ ensemble = #{ensemble,jdbcType=INTEGER},
+ write_quorum = #{writeQuorum,jdbcType=INTEGER},
+ ack_quorum = #{ackQuorum,jdbcType=INTEGER},
+ retention_time = #{retentionTime,jdbcType=INTEGER},
+ retention_time_unit = #{retentionTimeUnit,jdbcType=CHAR},
+ ttl = #{ttl,jdbcType=INTEGER},
+ ttl_unit = #{ttlUnit,jdbcType=VARCHAR},
+ retention_size = #{retentionSize,jdbcType=INTEGER},
+ retention_size_unit = #{retentionSizeUnit,jdbcType=VARCHAR},
+ is_deleted = #{isDeleted,jdbcType=INTEGER}
+ where id = #{id,jdbcType=INTEGER}
+ and is_deleted = 0
+ </update>
+ <update id="updateByIdentifierSelective" parameterType="org.apache.inlong.manager.dao.entity.BusinessPulsarEntity">
+ update business_pulsar
+ <set>
+ <if test="ensemble != null">
+ ensemble = #{ensemble,jdbcType=INTEGER},
+ </if>
+ <if test="writeQuorum != null">
+ write_quorum = #{writeQuorum,jdbcType=INTEGER},
+ </if>
+ <if test="ackQuorum != null">
+ ack_quorum = #{ackQuorum,jdbcType=INTEGER},
+ </if>
+ <if test="retentionTime != null">
+ retention_time = #{retentionTime,jdbcType=INTEGER},
+ </if>
+ <if test="retentionTimeUnit != null">
+ retention_time_unit = #{retentionTimeUnit,jdbcType=CHAR},
+ </if>
+ <if test="ttl != null">
+ ttl = #{ttl,jdbcType=INTEGER},
+ </if>
+ <if test="ttlUnit != null">
+ ttl_unit = #{ttlUnit,jdbcType=VARCHAR},
+ </if>
+ <if test="retentionSize != null">
+ retention_size = #{retentionSize,jdbcType=INTEGER},
+ </if>
+ <if test="retentionSizeUnit != null">
+ retention_size_unit = #{retentionSizeUnit,jdbcType=VARCHAR},
+ </if>
+ <if test="isDeleted != null">
+ is_deleted = #{isDeleted,jdbcType=INTEGER},
+ </if>
+ </set>
+ where inlong_group_id = #{inlongGroupId, jdbcType=VARCHAR}
+ and is_deleted = 0
+ </update>
+
+</mapper>
\ No newline at end of file
diff --git a/inlong-manager/manager-dao/src/main/resources/mappers/DataStreamEntityMapper.xml b/inlong-manager/manager-dao/src/main/resources/mappers/DataStreamEntityMapper.xml
index 70a0b38..759a0ac 100644
--- a/inlong-manager/manager-dao/src/main/resources/mappers/DataStreamEntityMapper.xml
+++ b/inlong-manager/manager-dao/src/main/resources/mappers/DataStreamEntityMapper.xml
@@ -171,6 +171,14 @@
and is_deleted = 0
</where>
</select>
+ <select id="selectTopicList" resultType="org.apache.inlong.manager.common.pojo.datastream.DataStreamTopicVO">
+ select inlong_stream_id, mq_resource_obj
+ from data_stream
+ <where>
+ inlong_group_id = #{groupId, jdbcType=VARCHAR}
+ and is_deleted = 0
+ </where>
+ </select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Integer">
delete
diff --git a/inlong-manager/manager-dao/src/test/resources/sql/apache_inlong_manager.sql b/inlong-manager/manager-dao/src/test/resources/sql/apache_inlong_manager.sql
index 1a88f06..9b8e124 100644
--- a/inlong-manager/manager-dao/src/test/resources/sql/apache_inlong_manager.sql
+++ b/inlong-manager/manager-dao/src/test/resources/sql/apache_inlong_manager.sql
@@ -27,7 +27,7 @@ CREATE TABLE `agent_heartbeat_log`
`ip` varchar(64) NOT NULL COMMENT 'agent host ip',
`version` varchar(128) DEFAULT NULL,
`heartbeat_msg` text DEFAULT NULL COMMENT 'massage in heartbeat request',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`ip`)
);
@@ -58,7 +58,7 @@ CREATE TABLE `agent_sys_conf`
`stat_interval_sec` int(11) NOT NULL DEFAULT '60' COMMENT 'Statistical message sending frequency',
`conf_refresh_interval_secs` int(11) NOT NULL DEFAULT '300' COMMENT 'The frequency at which the Agent regularly pulls the configuration from the InLongManager',
`flow_size` int(11) DEFAULT '1048576000',
- `bufferSize` int(11) DEFAULT NULL COMMENT 'bufferSize, default 1048576',
+ `bufferSize` int(11) DEFAULT '1048576' COMMENT 'bufferSize, default 1048576',
`compress` tinyint(2) DEFAULT NULL COMMENT 'Whether to compress',
`event_check_interval` int(11) DEFAULT NULL COMMENT 'File scanning period',
`is_calMD5` tinyint(2) DEFAULT NULL COMMENT 'Do you want to calculate the cumulative md5 of read characters',
@@ -71,46 +71,71 @@ CREATE TABLE `agent_sys_conf`
DROP TABLE IF EXISTS `business`;
CREATE TABLE `business`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
- `name` varchar(128) DEFAULT '' COMMENT 'Business name, English, numbers and underscore',
- `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'Business Introduction',
- `middleware_type` varchar(10) DEFAULT 'Tube' COMMENT 'The middleware type of data storage, high throughput: Tube',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, for Tube, its Topic',
- `daily_records` int(11) DEFAULT NULL COMMENT 'Number of access records per day, unit: 10,000 records per day',
- `daily_storage` int(11) DEFAULT NULL COMMENT 'Access size by day, unit: GB per day',
- `peak_records` int(11) DEFAULT NULL COMMENT 'Access peak per second, unit: records per second',
- `max_length` int(11) DEFAULT NULL COMMENT 'The maximum length of a single piece of data, unit: Byte',
- `schema_name` varchar(128) DEFAULT NULL COMMENT 'Data type, associated data_schema table',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
- `status` int(11) DEFAULT '21' COMMENT 'Business status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
+ `name` varchar(128) DEFAULT '' COMMENT 'Business name, English, numbers and underscore',
+ `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
+ `description` varchar(256) DEFAULT '' COMMENT 'Business Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `mq_resource_obj` varchar(128) NOT NULL COMMENT 'MQ resource object, for Tube, its Topic, for Pulsar, its Namespace',
+ `daily_records` int(11) DEFAULT '10' COMMENT 'Number of access records per day, unit: 10,000 records per day',
+ `daily_storage` int(11) DEFAULT '10' COMMENT 'Access size by day, unit: GB per day',
+ `peak_records` int(11) DEFAULT '1000' COMMENT 'Access peak per second, unit: records per second',
+ `max_length` int(11) DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte',
+ `schema_name` varchar(128) DEFAULT NULL COMMENT 'Data type, associated data_schema table',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
+ `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
+ `status` int(11) DEFAULT '21' COMMENT 'Business status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_business` (`inlong_group_id`, `is_deleted`, `modify_time`)
);
+
-- ----------------------------
--- Table structure for business_ext
+-- Table structure for business_pulsar
-- ----------------------------
-DROP TABLE IF EXISTS `business_ext`;
-CREATE TABLE `business_ext`
+DROP TABLE IF EXISTS `business_pulsar`;
+CREATE TABLE `business_pulsar`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'business descriptor',
- `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
+ `ensemble` int(3) DEFAULT '3' COMMENT 'The writable nodes number of ledger',
+ `write_quorum` int(3) DEFAULT '3' COMMENT 'The copies number of ledger',
+ `ack_quorum` int(3) DEFAULT '2' COMMENT 'The number of requested acks',
+ `retention_time` int(11) DEFAULT '72' COMMENT 'Message storage time',
+ `retention_time_unit` char(20) DEFAULT 'hours' COMMENT 'The unit of the message storage time',
+ `ttl` int(11) DEFAULT '24' COMMENT 'Message time-to-live duration',
+ `ttl_unit` varchar(20) DEFAULT 'hours' COMMENT 'The unit of time-to-live duration',
+ `retention_size` int(11) DEFAULT '-1' COMMENT 'Message size',
+ `retention_size_unit` varchar(20) DEFAULT 'MB' COMMENT 'The unit of message size',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
-- ----------------------------
+-- Table structure for business_ext
+-- ----------------------------
+DROP TABLE IF EXISTS `business_ext`;
+CREATE TABLE `business_ext`
+(
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id',
+ `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
+ `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ PRIMARY KEY (`id`),
+ KEY `index_group_id` (`inlong_group_id`)
+);
+
+-- ----------------------------
-- Table structure for cluster_info
-- ----------------------------
DROP TABLE IF EXISTS `cluster_info`;
@@ -118,19 +143,19 @@ CREATE TABLE `cluster_info`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`name` varchar(128) NOT NULL COMMENT 'cluster name',
- `type` varchar(32) DEFAULT NULL COMMENT 'Cluster type, including TUBE, ZOOKEEPER, etc.',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Cluster IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Cluster port number',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `type` varchar(32) NOT NULL COMMENT 'Cluster type, including TUBE, PULSAR, etc.',
+ `ip` varchar(64) NOT NULL COMMENT 'Cluster IP address',
+ `port` int(11) NOT NULL COMMENT 'Cluster port',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
`url` varchar(256) DEFAULT NULL COMMENT 'Cluster URL address',
`is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
`ext_props` text DEFAULT NULL COMMENT 'extended properties',
`status` int(11) DEFAULT '1' COMMENT 'cluster status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -140,32 +165,32 @@ CREATE TABLE `cluster_info`
DROP TABLE IF EXISTS `common_db_server`;
CREATE TABLE `common_db_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
- `db_type` varchar(128) DEFAULT NULL COMMENT 'DB type, such as MySQL, Oracle',
- `db_server_ip` varchar(64) DEFAULT NULL COMMENT 'DB Server IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `username` varchar(64) DEFAULT NULL COMMENT 'Username',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
- `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
- `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
- `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
- `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
- `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
- `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
- `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `connection_name` varchar(128) NOT NULL COMMENT 'The name of the database connection',
+ `db_type` varchar(128) DEFAULT 'MySQL' COMMENT 'DB type, such as MySQL, Oracle',
+ `db_server_ip` varchar(64) NOT NULL COMMENT 'DB Server IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
+ `username` varchar(64) NOT NULL COMMENT 'Username',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
+ `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
+ `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
+ `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
+ `in_charges` varchar(512) NOT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
+ `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
+ `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
+ `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
+ `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
);
@@ -175,22 +200,22 @@ CREATE TABLE `common_db_server`
DROP TABLE IF EXISTS `common_file_server`;
CREATE TABLE `common_file_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Data source IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
- `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
- `username` varchar(64) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy, LoadProxy',
+ `ip` varchar(64) NOT NULL COMMENT 'Data source IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
+ `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
+ `username` varchar(64) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
);
@@ -200,21 +225,21 @@ CREATE TABLE `common_file_server`
DROP TABLE IF EXISTS `consumption`;
CREATE TABLE `consumption`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `consumer_group_name` varchar(255) NOT NULL COMMENT 'consumer group name',
- `consumer_group_id` varchar(255) NOT NULL COMMENT 'Consumer group ID',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Person in charge of consumption',
- `inlong_group_id` varchar(255) DEFAULT NULL COMMENT 'Business ID',
- `middleware_type` varchar(64) DEFAULT NULL COMMENT 'The middleware type of data storage, high throughput: Tube',
- `topic` varchar(255) DEFAULT NULL COMMENT 'Consumption topic',
- `filter_enabled` int(2) DEFAULT NULL COMMENT 'Whether to filter',
- `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Consumption data stream ID',
- `status` int(11) NOT NULL COMMENT 'Status: draft, pending approval, approval rejected, approval passed',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
- `is_deleted` int(2) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `consumer_group_name` varchar(255) NOT NULL COMMENT 'consumer group name',
+ `consumer_group_id` varchar(255) NOT NULL COMMENT 'Consumer group ID',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Person in charge of consumption',
+ `inlong_group_id` varchar(255) NOT NULL COMMENT 'Business group id',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `topic` varchar(255) NOT NULL COMMENT 'Consumption topic',
+ `filter_enabled` int(2) DEFAULT '0' COMMENT 'Whether to filter, default 0, not filter consume',
+ `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Data stream ID for consumption, if filter_enable is 1, it cannot empty',
+ `status` int(11) NOT NULL COMMENT 'Status: draft, pending approval, approval rejected, approval passed',
+ `creator` varchar(64) NOT NULL COMMENT 'creator',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `is_deleted` int(2) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
PRIMARY KEY (`id`)
);
@@ -224,22 +249,22 @@ CREATE TABLE `consumption`
DROP TABLE IF EXISTS `data_proxy_cluster`;
CREATE TABLE `data_proxy_cluster`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `name` varchar(128) DEFAULT NULL COMMENT 'cluster name',
- `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
- `address` varchar(128) DEFAULT NULL COMMENT 'cluster address',
- `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
- `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
- `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `ext_props` text DEFAULT NULL COMMENT 'Extended properties',
- `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'Create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `name` varchar(128) NOT NULL COMMENT 'cluster name',
+ `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
+ `address` varchar(128) NOT NULL COMMENT 'cluster address',
+ `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
+ `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
+ `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
+ `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `ext_props` text DEFAULT NULL COMMENT 'Extended properties',
+ `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -275,7 +300,7 @@ CREATE TABLE `data_source_cmd_config`
`specified_data_time` varchar(64) NOT NULL,
`bSend` tinyint(1) NOT NULL,
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Last update time ',
- `create_time` timestamp NULL DEFAULT NULL,
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`result_info` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`)
);
@@ -287,11 +312,11 @@ DROP TABLE IF EXISTS `data_stream`;
CREATE TABLE `data_stream`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream descriptor, non-deleted globally unique',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `name` varchar(64) DEFAULT NULL COMMENT 'The name of the data stream page display, can be Chinese',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id, non-deleted globally unique',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `name` varchar(64) NOT NULL COMMENT 'The name of the data stream page display, can be Chinese',
`description` varchar(256) DEFAULT '' COMMENT 'Introduction to data stream',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id',
+ `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id, Pulsar is Topic',
`data_source_type` varchar(32) DEFAULT 'FILE' COMMENT 'Data source type, including: FILE, DB, Auto-Push (DATA_PROXY_SDK, HTTP)',
`storage_period` int(11) DEFAULT '1' COMMENT 'The storage period of data in MQ, unit: day',
`data_type` varchar(20) DEFAULT 'TEXT' COMMENT 'Data type, there are: TEXT, KEY-VALUE, PB, BON, TEXT and BON should be treated differently',
@@ -302,10 +327,10 @@ CREATE TABLE `data_stream`
`status` int(11) DEFAULT '0' COMMENT 'Data stream status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`)
);
@@ -316,13 +341,13 @@ CREATE TABLE `data_stream`
DROP TABLE IF EXISTS `data_stream_ext`;
CREATE TABLE `data_stream_ext`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream descriptor',
- `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
+ `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
+ `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -332,23 +357,24 @@ CREATE TABLE `data_stream_ext`
DROP TABLE IF EXISTS `data_stream_field`;
CREATE TABLE `data_stream_field`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream descriptor',
- `is_predefined_field` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a predefined field, 0: no, 1: yes',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
- `field_value` varchar(128) DEFAULT NULL COMMENT 'Field value, required if it is a predefined field',
- `pre_expression` varchar(256) DEFAULT NULL COMMENT 'Pre-defined field value expression',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
- `field_comment` varchar(50) DEFAULT NULL COMMENT 'Field description',
- `rank_num` smallint(6) DEFAULT '0' COMMENT 'Field order (front-end display field order)',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `is_exist` tinyint(1) DEFAULT '0' COMMENT 'Does it exist, 0: does not exist, 1: exists',
- `bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
- `bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
- `encrypt_level` varchar(20) DEFAULT NULL COMMENT 'Encryption level',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
+ `is_predefined_field` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a predefined field, 0: no, 1: yes',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
+ `field_value` varchar(128) DEFAULT NULL COMMENT 'Field value, required if it is a predefined field',
+ `pre_expression` varchar(256) DEFAULT NULL COMMENT 'Pre-defined field value expression',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
+ `field_comment` varchar(50) DEFAULT NULL COMMENT 'Field description',
+ `rank_num` smallint(6) DEFAULT '0' COMMENT 'Field order (front-end display field order)',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `is_exist` tinyint(1) DEFAULT '0' COMMENT 'Does it exist, 0: does not exist, 1: exists',
+ `bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
+ `bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
+ `encrypt_level` varchar(20) DEFAULT NULL COMMENT 'Encryption level',
PRIMARY KEY (`id`)
);
+
-- ----------------------------
-- Table structure for operation_log
-- ----------------------------
@@ -380,8 +406,8 @@ DROP TABLE IF EXISTS `role`;
CREATE TABLE `role`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
- `role_code` varchar(100) NOT NULL COMMENT 'role code',
- `role_name` varchar(255) NOT NULL COMMENT 'role Chinese name',
+ `role_code` varchar(100) NOT NULL COMMENT 'Role code',
+ `role_name` varchar(255) NOT NULL COMMENT 'Role Chinese name',
`create_time` datetime NOT NULL,
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_by` varchar(255) NOT NULL,
@@ -396,16 +422,16 @@ CREATE TABLE `role`
DROP TABLE IF EXISTS `source_db_basic`;
CREATE TABLE `source_db_basic`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream descriptor',
- `sync_type` tinyint(1) DEFAULT '0' COMMENT 'Data synchronization type, 0: FULL, full amount, 1: INCREMENTAL, incremental',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
+ `sync_type` tinyint(1) DEFAULT '0' COMMENT 'Data synchronization type, 0: FULL, full amount, 1: INCREMENTAL, incremental',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`)
);
@@ -415,25 +441,25 @@ CREATE TABLE `source_db_basic`
DROP TABLE IF EXISTS `source_db_detail`;
CREATE TABLE `source_db_detail`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream descriptor',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'database name',
- `transfer_ip` varchar(64) DEFAULT NULL COMMENT 'Transfer IP',
- `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
- `table_name` varchar(128) DEFAULT NULL COMMENT 'Data table name, required for increment',
- `table_fields` longtext COMMENT 'Data table fields, multiple are separated by half-width commas, required for increment',
- `data_sql` longtext COMMENT 'SQL statement to collect source data, required for full amount',
- `crontab` varchar(56) DEFAULT NULL COMMENT 'Timed scheduling expression, required for full amount',
- `status` int(11) DEFAULT '0' COMMENT 'Data source status',
- `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save unsubmitted and unapproved intermediate data after modification',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `db_name` varchar(128) DEFAULT NULL COMMENT 'database name',
+ `transfer_ip` varchar(64) DEFAULT NULL COMMENT 'Transfer IP',
+ `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
+ `table_name` varchar(128) DEFAULT NULL COMMENT 'Data table name, required for increment',
+ `table_fields` longtext COMMENT 'Data table fields, multiple are separated by half-width commas, required for increment',
+ `data_sql` longtext COMMENT 'SQL statement to collect source data, required for full amount',
+ `crontab` varchar(56) DEFAULT NULL COMMENT 'Timed scheduling expression, required for full amount',
+ `status` int(11) DEFAULT '0' COMMENT 'Data source status',
+ `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
);
@@ -443,22 +469,22 @@ CREATE TABLE `source_db_detail`
DROP TABLE IF EXISTS `source_file_basic`;
CREATE TABLE `source_file_basic`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'business group id',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'data stream id',
- `is_hybrid_source` tinyint(1) DEFAULT '0' COMMENT 'Whether to mix data sources',
- `is_table_mapping` tinyint(1) DEFAULT '0' COMMENT 'Is there a table name mapping',
- `date_offset` int(4) DEFAULT '0' COMMENT 'Time offset\n',
- `date_offset_unit` varchar(2) DEFAULT 'H' COMMENT 'time offset unit',
- `file_rolling_type` varchar(2) DEFAULT 'H' COMMENT 'file rolling type',
- `upload_max_size` int(4) DEFAULT '120' COMMENT 'upload maximum size',
- `need_compress` tinyint(1) DEFAULT '0' COMMENT 'whether need compress',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'delete switch',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'temp view',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'ID',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id',
+ `is_hybrid_source` tinyint(1) DEFAULT '0' COMMENT 'Whether to mix data sources',
+ `is_table_mapping` tinyint(1) DEFAULT '0' COMMENT 'Is there a table name mapping',
+ `date_offset` int(4) DEFAULT '0' COMMENT 'Time offset\n',
+ `date_offset_unit` varchar(2) DEFAULT 'H' COMMENT 'Time offset unit',
+ `file_rolling_type` varchar(2) DEFAULT 'H' COMMENT 'File rolling type',
+ `upload_max_size` int(4) DEFAULT '120' COMMENT 'Upload maximum size',
+ `need_compress` tinyint(1) DEFAULT '0' COMMENT 'Whether need compress',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Delete switch',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'temp view',
PRIMARY KEY (`id`)
);
@@ -468,26 +494,26 @@ CREATE TABLE `source_file_basic`
DROP TABLE IF EXISTS `source_file_detail`;
CREATE TABLE `source_file_detail`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream descriptor',
- `access_type` varchar(20) DEFAULT 'Agent' COMMENT 'Collection type, there are Agent, DataProxy client, LoadProxy, the file can only be Agent temporarily',
- `server_name` varchar(64) DEFAULT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
- `ip` varchar(128) DEFAULT NULL COMMENT 'Data source IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Data source port number',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
- `issue_type` varchar(10) DEFAULT 'SSH' COMMENT 'Issuing method, there are SSH, TCS',
- `username` varchar(32) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `file_path` varchar(256) DEFAULT NULL COMMENT 'File path, supports regular matching',
- `status` int(11) DEFAULT '0' COMMENT 'Data source status',
- `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save unsubmitted and unapproved intermediate data after modification',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
+ `access_type` varchar(20) DEFAULT 'Agent' COMMENT 'Collection type, there are Agent, DataProxy client, LoadProxy, the file can only be Agent temporarily',
+ `server_name` varchar(64) NOT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
+ `ip` varchar(128) NOT NULL COMMENT 'Data source IP address',
+ `port` int(11) NOT NULL COMMENT 'Data source port number',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
+ `issue_type` varchar(10) DEFAULT 'SSH' COMMENT 'Issuing method, there are SSH, TCS',
+ `username` varchar(32) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `file_path` varchar(256) NOT NULL COMMENT 'File path, supports regular matching',
+ `status` int(11) DEFAULT '0' COMMENT 'Data source status',
+ `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
);
@@ -514,33 +540,33 @@ DROP TABLE IF EXISTS `storage_hive`;
DROP TABLE IF EXISTS `storage_hive`;
CREATE TABLE `storage_hive`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business descriptor',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream descriptor',
- `jdbc_url` varchar(255) DEFAULT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
- `username` varchar(128) DEFAULT NULL COMMENT 'Username',
- `password` varchar(255) DEFAULT NULL COMMENT 'User password',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `table_name` varchar(128) DEFAULT NULL COMMENT 'Target data table name',
- `primary_partition` varchar(255) DEFAULT 'dt' COMMENT 'primary partition field',
- `secondary_partition` varchar(256) DEFAULT NULL COMMENT 'secondary partition field',
- `partition_type` varchar(10) DEFAULT NULL COMMENT 'The partition type, there are: H-by hour, D-by day, W-by week, M-by month, O-one-time, R-non-periodical',
- `file_format` varchar(15) DEFAULT 'TextFile' COMMENT 'The stored table format, TextFile, RCFile, SequenceFile, Avro',
- `encoding_type` varchar(255) DEFAULT NULL COMMENT 'data encoding',
- `field_splitter` varchar(10) DEFAULT NULL COMMENT 'field separator',
- `hdfs_default_fs` varchar(255) DEFAULT NULL COMMENT 'HDFS defaultFS, such as "hdfs://127.0.0.1:9000"',
- `warehouse_dir` varchar(250) DEFAULT '/user/hive/warehouse' COMMENT 'Hive table storage path on HDFS, such as "/user/hive/warehouse"',
- `usage_interval` varchar(10) DEFAULT NULL COMMENT 'The amount of time that Sort collected data will land on Hive, there are 10M, 15M, 30M, 1H, 1D',
- `storage_period` int(5) DEFAULT '10' COMMENT 'Data storage period, unit: day',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
- `opt_log` varchar(5000) DEFAULT NULL COMMENT 'Background operation log',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
+ `jdbc_url` varchar(255) NOT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
+ `username` varchar(128) NOT NULL COMMENT 'Username',
+ `password` varchar(255) NOT NULL COMMENT 'User password',
+ `db_name` varchar(128) NOT NULL COMMENT 'Target database name',
+ `table_name` varchar(128) NOT NULL COMMENT 'Target data table name',
+ `primary_partition` varchar(255) DEFAULT 'dt' COMMENT 'primary partition field',
+ `secondary_partition` varchar(256) DEFAULT NULL COMMENT 'secondary partition field',
+ `partition_type` varchar(10) DEFAULT NULL COMMENT 'The partition type, there are: H-by hour, D-by day, W-by week, M-by month, O-one-time, R-non-periodical',
+ `file_format` varchar(15) DEFAULT 'TextFile' COMMENT 'The stored table format, TextFile, RCFile, SequenceFile, Avro',
+ `encoding_type` varchar(255) DEFAULT NULL COMMENT 'data encoding',
+ `field_splitter` varchar(10) DEFAULT NULL COMMENT 'field separator',
+ `hdfs_default_fs` varchar(255) DEFAULT NULL COMMENT 'HDFS defaultFS, such as "hdfs://127.0.0.1:9000"',
+ `warehouse_dir` varchar(250) DEFAULT '/user/hive/warehouse' COMMENT 'Hive table storage path on HDFS, such as "/user/hive/warehouse"',
+ `usage_interval` varchar(10) DEFAULT NULL COMMENT 'The amount of time that Sort collected data will land on Hive, there are 10M, 15M, 30M, 1H, 1D',
+ `storage_period` int(5) DEFAULT '10' COMMENT 'Data storage period, unit: day',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
+ `opt_log` varchar(5000) DEFAULT NULL COMMENT 'Background operation log',
PRIMARY KEY (`id`)
);
@@ -550,13 +576,13 @@ CREATE TABLE `storage_hive`
DROP TABLE IF EXISTS `storage_hive_field`;
CREATE TABLE `storage_hive_field`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
- `source_field_name` varchar(20) DEFAULT NULL COMMENT 'source field name',
- `source_field_type` varchar(20) DEFAULT NULL COMMENT 'source field type',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
- `field_comment` varchar(2000) DEFAULT NULL COMMENT 'Field description',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
+ `source_field_name` varchar(20) NOT NULL COMMENT 'source field name',
+ `source_field_type` varchar(20) NOT NULL COMMENT 'source field type',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
+ `field_comment` varchar(2000) DEFAULT '' COMMENT 'Field description',
`is_required` tinyint(1) DEFAULT NULL COMMENT 'Is it required, 0: not necessary, 1: required',
`bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
`bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
@@ -573,15 +599,15 @@ CREATE TABLE `storage_hive_field`
DROP TABLE IF EXISTS `task`;
CREATE TABLE `task`
(
- `id` bigint(20) NOT NULL,
- `taskflow_id` bigint(20) DEFAULT NULL COMMENT 'Owning task flow id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_id` bigint(20) NOT NULL COMMENT 'Owning task flow id',
`task_def_id` bigint(20) DEFAULT NULL COMMENT 'task definition id',
- `task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
+ `task_name` varchar(255) NOT NULL COMMENT 'task name',
`status` varchar(255) DEFAULT NULL COMMENT 'task status',
`post_param` varchar(255) DEFAULT NULL COMMENT 'Task parameters',
`resultmsg` varchar(1000) DEFAULT NULL COMMENT 'Execution result log',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(0) DEFAULT NULL COMMENT 'last modified person',
PRIMARY KEY (`id`)
@@ -593,13 +619,13 @@ CREATE TABLE `task`
DROP TABLE IF EXISTS `task_def`;
CREATE TABLE `task_def`
(
- `id` bigint(20) NOT NULL,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Task flow definition id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Task flow definition id',
`parent_id` bigint(20) DEFAULT NULL COMMENT 'parent task id',
`implclass` varchar(255) DEFAULT NULL COMMENT 'task processing flow class',
`task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` datetime DEFAULT NULL COMMENT 'last modified person',
`delivery_id` bigint(20) DEFAULT NULL COMMENT 'Task push method',
@@ -612,11 +638,11 @@ CREATE TABLE `task_def`
DROP TABLE IF EXISTS `taskflow`;
CREATE TABLE `taskflow`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Taskflow definition id',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Taskflow definition id',
`status` varchar(255) DEFAULT NULL COMMENT 'status',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
+ `create_time` datetime DEFAULT NULL COMMENT 'Create time',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(255) DEFAULT NULL COMMENT 'last modified person',
`event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
@@ -629,11 +655,11 @@ CREATE TABLE `taskflow`
DROP TABLE IF EXISTS `taskflow_def`;
CREATE TABLE `taskflow_def`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) DEFAULT NULL COMMENT 'Workflow definition name',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) NOT NULL COMMENT 'Workflow definition name',
`descrip` varchar(255) DEFAULT NULL COMMENT 'Workflow function description',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`isValid` int(11) DEFAULT NULL COMMENT 'logical deletion',
`trigger_event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
PRIMARY KEY (`id`)
@@ -650,7 +676,7 @@ CREATE TABLE `user`
`password` varchar(64) NOT NULL COMMENT 'password md5',
`account_type` int(11) NOT NULL DEFAULT '1' COMMENT 'account type, 0-manager 1-normal',
`due_date` datetime DEFAULT NULL COMMENT 'due date for account',
- `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
`create_by` varchar(255) NOT NULL COMMENT 'create by sb.',
`update_by` varchar(255) DEFAULT NULL COMMENT 'update by sb.',
@@ -694,7 +720,7 @@ CREATE TABLE `wf_approver`
`approvers` varchar(1024) NOT NULL COMMENT 'Approvers, separated by commas',
`creator` varchar(64) NOT NULL COMMENT 'creator',
`modifier` varchar(64) NOT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
`is_deleted` int(11) DEFAULT '0' COMMENT 'Whether to delete, 0 is not deleted, if greater than 0, delete',
PRIMARY KEY (`id`)
@@ -718,8 +744,8 @@ CREATE TABLE `wf_event_log`
`process_inst_id` int(11) NOT NULL,
`process_name` varchar(255) DEFAULT NULL COMMENT 'Process name',
`process_display_name` varchar(255) NOT NULL COMMENT 'Process name',
- `group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id',
- `task_inst_id` int(11) DEFAULT NULL COMMENT 'Task id',
+ `inlong_group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id',
+ `task_inst_id` int(11) DEFAULT NULL COMMENT 'Task ID',
`element_name` varchar(255) NOT NULL COMMENT 'The name of the component that triggered the event',
`element_display_name` varchar(255) NOT NULL COMMENT 'Chinese name of the component that triggered the event',
`event_type` varchar(64) NOT NULL COMMENT 'Event type: process event/task event',
@@ -741,19 +767,19 @@ CREATE TABLE `wf_event_log`
DROP TABLE IF EXISTS `wf_process_instance`;
CREATE TABLE `wf_process_instance`
(
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL COMMENT 'process name',
- `display_name` varchar(255) NOT NULL COMMENT 'Process display name',
- `type` varchar(255) DEFAULT NULL COMMENT 'Process classification',
- `title` varchar(255) DEFAULT NULL COMMENT 'Process title',
- `group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id: to facilitate related business',
- `applicant` varchar(255) NOT NULL COMMENT 'applicant',
- `state` varchar(64) NOT NULL COMMENT 'state',
- `form_data` mediumtext COMMENT 'form information',
- `start_time` datetime NOT NULL COMMENT 'start time',
- `end_time` datetime DEFAULT NULL COMMENT 'End event',
- `ext` text COMMENT 'Extended information-json',
- `hidden` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Is it hidden',
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) NOT NULL COMMENT 'process name',
+ `display_name` varchar(255) NOT NULL COMMENT 'Process display name',
+ `type` varchar(255) DEFAULT NULL COMMENT 'Process classification',
+ `title` varchar(255) DEFAULT NULL COMMENT 'Process title',
+ `inlong_group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id: to facilitate related business',
+ `applicant` varchar(255) NOT NULL COMMENT 'applicant',
+ `state` varchar(64) NOT NULL COMMENT 'state',
+ `form_data` mediumtext COMMENT 'form information',
+ `start_time` datetime NOT NULL COMMENT 'start time',
+ `end_time` datetime DEFAULT NULL COMMENT 'End event',
+ `ext` text COMMENT 'Extended information-text',
+ `hidden` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Is it hidden',
PRIMARY KEY (`id`)
);
@@ -778,7 +804,7 @@ CREATE TABLE `wf_task_instance`
`form_data` mediumtext COMMENT 'form information submitted by the current task',
`start_time` datetime NOT NULL COMMENT 'start time',
`end_time` datetime DEFAULT NULL COMMENT 'End time',
- `ext` text COMMENT 'Extended information-json',
+ `ext` text COMMENT 'Extended information-text',
PRIMARY KEY (`id`)
);
@@ -790,17 +816,17 @@ CREATE TABLE `cluster_set`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
- `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'ClusterSet Introduction',
- `middleware_type` varchar(10) DEFAULT 'Pulsar' COMMENT 'The middleware type of data storage, high throughput: Pulsar',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
+ `cn_name` varchar(256) COMMENT 'Chinese display name',
+ `description` varchar(256) COMMENT 'ClusterSet Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `in_charges` varchar(512) COMMENT 'Name of responsible person, separated by commas',
+ `followers` varchar(512) COMMENT 'List of names of business followers, separated by commas',
`status` int(11) DEFAULT '21' COMMENT 'ClusterSet status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -810,8 +836,8 @@ CREATE TABLE `cluster_set`
DROP TABLE IF EXISTS `cluster_set_inlongid`;
CREATE TABLE `cluster_set_inlongid`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
PRIMARY KEY (`id`)
);
@@ -838,7 +864,7 @@ CREATE TABLE `cache_cluster_ext`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`cluster_name` varchar(128) NOT NULL COMMENT 'CacheCluster name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
@@ -907,7 +933,7 @@ CREATE TABLE `flume_source_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSource name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
@@ -936,7 +962,7 @@ CREATE TABLE `flume_channel_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeChannel name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
@@ -966,10 +992,10 @@ CREATE TABLE `flume_sink_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSink name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
);
-SET FOREIGN_KEY_CHECKS = 1;
\ No newline at end of file
+SET FOREIGN_KEY_CHECKS = 1;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/DataStreamService.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/DataStreamService.java
index 041c3cd..f23a722 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/DataStreamService.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/DataStreamService.java
@@ -25,6 +25,7 @@ import org.apache.inlong.manager.common.pojo.datastream.DataStreamInfoToHiveConf
import org.apache.inlong.manager.common.pojo.datastream.DataStreamListVO;
import org.apache.inlong.manager.common.pojo.datastream.DataStreamPageRequest;
import org.apache.inlong.manager.common.pojo.datastream.DataStreamSummaryInfo;
+import org.apache.inlong.manager.common.pojo.datastream.DataStreamTopicVO;
import org.apache.inlong.manager.common.pojo.datastream.FullPageInfo;
import org.apache.inlong.manager.common.pojo.datastream.FullPageUpdateInfo;
@@ -162,6 +163,11 @@ public interface DataStreamService {
int selectCountByGroupId(String groupId);
/**
+ * According to the business group id, query the Topic list
+ */
+ List<DataStreamTopicVO> getTopicList(String groupId);
+
+ /**
* Save the information modified when the approval is passed
*
* @param streamApproveList data stream approval information
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/BusinessServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/BusinessServiceImpl.java
index 4d68b5c..1776843 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/BusinessServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/BusinessServiceImpl.java
@@ -37,13 +37,16 @@ import org.apache.inlong.manager.common.pojo.business.BusinessExtInfo;
import org.apache.inlong.manager.common.pojo.business.BusinessInfo;
import org.apache.inlong.manager.common.pojo.business.BusinessListVO;
import org.apache.inlong.manager.common.pojo.business.BusinessPageRequest;
+import org.apache.inlong.manager.common.pojo.business.BusinessPulsarInfo;
import org.apache.inlong.manager.common.pojo.business.BusinessTopicVO;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.entity.BusinessEntity;
import org.apache.inlong.manager.dao.entity.BusinessExtEntity;
+import org.apache.inlong.manager.dao.entity.BusinessPulsarEntity;
import org.apache.inlong.manager.dao.mapper.BusinessEntityMapper;
import org.apache.inlong.manager.dao.mapper.BusinessExtEntityMapper;
+import org.apache.inlong.manager.dao.mapper.BusinessPulsarEntityMapper;
import org.apache.inlong.manager.service.core.BusinessService;
import org.apache.inlong.manager.service.core.DataStreamService;
import org.slf4j.Logger;
@@ -59,7 +62,8 @@ import org.springframework.transaction.annotation.Transactional;
public class BusinessServiceImpl implements BusinessService {
private static final Logger LOGGER = LoggerFactory.getLogger(BusinessServiceImpl.class);
-
+ @Autowired
+ BusinessPulsarEntityMapper businessPulsarMapper;
@Autowired
private BusinessEntityMapper businessMapper;
@Autowired
@@ -96,15 +100,48 @@ public class BusinessServiceImpl implements BusinessService {
// After saving, the status is set to [BIZ_WAIT_SUBMIT]
entity.setStatus(EntityStatus.BIZ_WAIT_SUBMIT.getCode());
-
+ entity.setIsDeleted(EntityStatus.UN_DELETED.getCode());
entity.setCreator(operator);
entity.setModifier(operator);
entity.setCreateTime(new Date());
businessMapper.insertSelective(entity);
-
this.saveExt(groupId, businessInfo.getExtList());
- LOGGER.info("success to save business info");
+ if (BizConstant.MIDDLEWARE_PULSAR.equals(businessInfo.getMiddlewareType())) {
+ BusinessPulsarInfo pulsarInfo = (BusinessPulsarInfo) businessInfo.getMqExtInfo();
+ Preconditions.checkNotNull(pulsarInfo, "Pulsar info cannot be empty, as the middleware is Pulsar");
+
+ // Pulsar params must meet: ackQuorum <= writeQuorum <= ensemble
+ Integer ackQuorum = pulsarInfo.getAckQuorum();
+ Integer writeQuorum = pulsarInfo.getWriteQuorum();
+
+ Preconditions.checkNotNull(ackQuorum, "Pulsar ackQuorum cannot be empty");
+ Preconditions.checkNotNull(writeQuorum, "Pulsar writeQuorum cannot be empty");
+
+ if (!(ackQuorum <= writeQuorum)) {
+ throw new BusinessException(BizErrorCodeEnum.BUSINESS_SAVE_FAILED,
+ "Pulsar params must meet: ackQuorum <= writeQuorum");
+ }
+ // The default value of ensemble is writeQuorum
+ pulsarInfo.setEnsemble(writeQuorum);
+
+ // Pulsar entity may already exist, such as unsuccessfully deleted, or modify the business MQ type to Tube,
+ // need to delete and add the Pulsar entity with the same group id
+ BusinessPulsarEntity pulsarEntity = businessPulsarMapper.selectByGroupId(groupId);
+ if (pulsarEntity == null) {
+ pulsarEntity = CommonBeanUtils.copyProperties(pulsarInfo, BusinessPulsarEntity::new);
+ pulsarEntity.setIsDeleted(0);
+ pulsarEntity.setInlongGroupId(groupId);
+ businessPulsarMapper.insertSelective(pulsarEntity);
+ } else {
+ Integer id = pulsarEntity.getId();
+ pulsarEntity = CommonBeanUtils.copyProperties(pulsarInfo, BusinessPulsarEntity::new);
+ pulsarEntity.setId(id);
+ businessPulsarMapper.updateByPrimaryKeySelective(pulsarEntity);
+ }
+ }
+
+ LOGGER.info("success to save business info for groupId={}", groupId);
return groupId;
}
@@ -124,7 +161,26 @@ public class BusinessServiceImpl implements BusinessService {
.copyListProperties(extEntityList, BusinessExtInfo::new);
businessInfo.setExtList(extInfoList);
- LOGGER.info("success to get business info");
+ // If the middleware is Pulsar, we need to encapsulate Pulsar related data
+ String middlewareType = entity.getMiddlewareType();
+ if (BizConstant.MIDDLEWARE_PULSAR.equalsIgnoreCase(middlewareType)) {
+ BusinessPulsarEntity pulsarEntity = businessPulsarMapper.selectByGroupId(groupId);
+ Preconditions.checkNotNull(pulsarEntity, "Pulsar info not found for the Pulsar business");
+ BusinessPulsarInfo pulsarInfo = CommonBeanUtils.copyProperties(pulsarEntity, BusinessPulsarInfo::new);
+ businessInfo.setMqExtInfo(pulsarInfo);
+ }
+
+ // For approved business, encapsulate the cluster address of the middleware
+ if (EntityStatus.BIZ_CONFIG_SUCCESSFUL.getCode().equals(businessInfo.getStatus())) {
+ if (BizConstant.MIDDLEWARE_TUBE.equalsIgnoreCase(middlewareType)) {
+ businessInfo.setTubeMaster(clusterBean.getTubeMaster());
+ } else if (BizConstant.MIDDLEWARE_PULSAR.equalsIgnoreCase(middlewareType)) {
+ businessInfo.setPulsarAdminUrl(clusterBean.getPulsarAdminUrl());
+ businessInfo.setPulsarServiceUrl(clusterBean.getPulsarServiceUrl());
+ }
+ }
+
+ LOGGER.info("success to get business info for groupId={}", groupId);
return businessInfo;
}
@@ -170,7 +226,21 @@ public class BusinessServiceImpl implements BusinessService {
// Save extended information
this.updateExt(groupId, businessInfo.getExtList());
- LOGGER.info("success to update business info");
+ // Update the Pulsar info
+ if (BizConstant.MIDDLEWARE_PULSAR.equals(businessInfo.getMiddlewareType())) {
+ BusinessPulsarInfo pulsarInfo = (BusinessPulsarInfo) businessInfo.getMqExtInfo();
+ Preconditions.checkNotNull(pulsarInfo, "Pulsar info cannot be empty, as the middleware is Pulsar");
+ Integer writeQuorum = pulsarInfo.getWriteQuorum();
+ Integer ackQuorum = pulsarInfo.getAckQuorum();
+ if (!(ackQuorum <= writeQuorum)) {
+ throw new BusinessException(BizErrorCodeEnum.BUSINESS_SAVE_FAILED,
+ "Pulsar params must meet: ackQuorum <= writeQuorum");
+ }
+ BusinessPulsarEntity pulsarEntity = CommonBeanUtils.copyProperties(pulsarInfo, BusinessPulsarEntity::new);
+ businessPulsarMapper.updateByIdentifierSelective(pulsarEntity);
+ }
+
+ LOGGER.info("success to update business info for groupId={}", groupId);
return groupId;
}
@@ -219,7 +289,7 @@ public class BusinessServiceImpl implements BusinessService {
businessMapper.updateStatusByIdentifier(groupId, status, operator);
- LOGGER.info("success to update business status");
+ LOGGER.info("success to update business status for groupId={}", groupId);
return true;
}
@@ -262,10 +332,12 @@ public class BusinessServiceImpl implements BusinessService {
businessMapper.updateByIdentifierSelective(entity);
// To logically delete the associated extension table
- LOGGER.debug("begin to delete business ext property, groupId={}", groupId);
businessExtMapper.logicDeleteAllByGroupId(groupId);
- LOGGER.info("success to delete business and business ext property");
+ // To logically delete the associated pulsar table
+ businessPulsarMapper.logicDeleteByGroupId(groupId);
+
+ LOGGER.info("success to delete business and business ext property for groupId={}", groupId);
return true;
}
@@ -297,7 +369,7 @@ public class BusinessServiceImpl implements BusinessService {
countVO.setRejectCount(countVO.getRejectCount() + count);
}
}
- LOGGER.info("success to count business");
+ LOGGER.info("success to count business for operator={}", operator);
return countVO;
}
@@ -309,12 +381,17 @@ public class BusinessServiceImpl implements BusinessService {
String middlewareType = businessInfo.getMiddlewareType();
BusinessTopicVO topicVO = new BusinessTopicVO();
- if (BizConstant.MIDDLEWARE_TYPE_TUBE.equalsIgnoreCase(middlewareType)) {
+ if (BizConstant.MIDDLEWARE_TUBE.equalsIgnoreCase(middlewareType)) {
// Tube Topic corresponds to business one-to-one
topicVO.setTopicName(businessInfo.getMqResourceObj());
- topicVO.setMasterUrl(clusterBean.getTubeMaster());
+ topicVO.setTubeMasterUrl(clusterBean.getTubeMaster());
+ } else if (BizConstant.MIDDLEWARE_PULSAR.equalsIgnoreCase(middlewareType)) {
+ // Pulsar's topic corresponds to the data stream one-to-one
+ topicVO.setDsTopicList(streamService.getTopicList(groupId));
+ topicVO.setPulsarAdminUrl(clusterBean.getPulsarAdminUrl());
+ topicVO.setPulsarServiceUrl(clusterBean.getPulsarServiceUrl());
} else {
- LOGGER.error("middlewareType={} not supported", middlewareType);
+ LOGGER.error("middleware type={} not supported", middlewareType);
throw new BusinessException(BizErrorCodeEnum.MIDDLEWARE_TYPE_NOT_SUPPORTED);
}
@@ -332,16 +409,54 @@ public class BusinessServiceImpl implements BusinessService {
Preconditions.checkNotNull(approveInfo, "BusinessApproveInfo is empty");
String groupId = approveInfo.getInlongGroupId();
Preconditions.checkNotNull(groupId, BizConstant.GROUP_ID_IS_EMPTY);
+ String middlewareType = approveInfo.getMiddlewareType();
+ Preconditions.checkNotNull(middlewareType, "Middleware type is empty");
// Update status to [BIZ_CONFIG_ING]
// If you need to change business info after approve, just do in here
this.updateStatus(groupId, EntityStatus.BIZ_CONFIG_ING.getCode(), operator);
- LOGGER.info("success to update business status after approve");
+ if (BizConstant.MIDDLEWARE_PULSAR.equalsIgnoreCase(middlewareType)) {
+ BusinessPulsarEntity pulsarEntity = checkAndGetEntity(approveInfo);
+ businessPulsarMapper.updateByIdentifierSelective(pulsarEntity);
+ }
+
+ LOGGER.info("success to update business status after approve for groupId={}", groupId);
return true;
}
/**
+ * Check whether the Pulsar parameters filled in during approval are valid,
+ * if valid, return to the encapsulated entity
+ */
+ private BusinessPulsarEntity checkAndGetEntity(BusinessApproveInfo approveInfo) {
+ // Pulsar params must meet: ackQuorum <= writeQuorum <= ensemble
+ Integer ackQuorum = approveInfo.getAckQuorum();
+ Integer writeQuorum = approveInfo.getWriteQuorum();
+ Integer ensemble = approveInfo.getEnsemble();
+ Preconditions.checkNotNull(ackQuorum, "Pulsar ackQuorum cannot be empty");
+ Preconditions.checkNotNull(writeQuorum, "Pulsar writeQuorum cannot be empty");
+ Preconditions.checkNotNull(ensemble, "Pulsar ensemble cannot be empty");
+ if (!(ackQuorum <= writeQuorum && writeQuorum <= ensemble)) {
+ throw new BusinessException(BizErrorCodeEnum.BUSINESS_SAVE_FAILED,
+ "Pulsar params must meet: ackQuorum <= writeQuorum <= ensemble");
+ }
+
+ Preconditions.checkTrue(approveInfo.getTopicPartitionNum() != null
+ && approveInfo.getTopicPartitionNum() >= 1 && approveInfo.getTopicPartitionNum() <= 20,
+ "topic partition num must meet >= 1 and <= 20");
+
+ Preconditions.checkTrue(approveInfo.getTtl() != null && approveInfo.getTtlUnit() != null,
+ "retention size and unit cannot be empty");
+ Preconditions.checkTrue(approveInfo.getRetentionSize() != null && approveInfo.getRetentionSizeUnit() != null,
+ "retention size and unit cannot be empty");
+ Preconditions.checkTrue(approveInfo.getRetentionTime() != null && approveInfo.getRetentionTimeUnit() != null,
+ "retention size and unit cannot be empty");
+
+ return CommonBeanUtils.copyProperties(approveInfo, BusinessPulsarEntity::new);
+ }
+
+ /**
* Update extended information
* <p/>First physically delete the existing extended information, and then add this batch of extended information
*/
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataStreamServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataStreamServiceImpl.java
index 108d95c..7754a39 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataStreamServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/DataStreamServiceImpl.java
@@ -46,6 +46,7 @@ import org.apache.inlong.manager.common.pojo.datastream.DataStreamInfoToHiveConf
import org.apache.inlong.manager.common.pojo.datastream.DataStreamListVO;
import org.apache.inlong.manager.common.pojo.datastream.DataStreamPageRequest;
import org.apache.inlong.manager.common.pojo.datastream.DataStreamSummaryInfo;
+import org.apache.inlong.manager.common.pojo.datastream.DataStreamTopicVO;
import org.apache.inlong.manager.common.pojo.datastream.FullPageInfo;
import org.apache.inlong.manager.common.pojo.datastream.FullPageUpdateInfo;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
@@ -125,7 +126,7 @@ public class DataStreamServiceImpl implements DataStreamService {
// Process data source fields
this.saveField(groupId, streamId, dataStreamInfo.getFieldList());
- LOGGER.info("success to save data stream info");
+ LOGGER.info("success to save data stream info for groupId={}", groupId);
return streamEntity.getId();
}
@@ -144,7 +145,7 @@ public class DataStreamServiceImpl implements DataStreamService {
DataStreamInfo streamInfo = CommonBeanUtils.copyProperties(streamEntity, DataStreamInfo::new);
this.setStreamExtAndField(groupId, streamId, streamInfo);
- LOGGER.info("success to get data stream");
+ LOGGER.info("success to get data stream for groupId={}", groupId);
return streamInfo;
}
@@ -207,7 +208,7 @@ public class DataStreamServiceImpl implements DataStreamService {
PageInfo<DataStreamListVO> page = new PageInfo<>(dataStreamList);
page.setTotal(dataStreamList.size());
- LOGGER.debug("success to list data stream info");
+ LOGGER.debug("success to list data stream info for groupId={}", groupId);
return page;
}
@@ -242,7 +243,7 @@ public class DataStreamServiceImpl implements DataStreamService {
this.updateField(groupId, streamId, streamInfo.getFieldList());
}
- LOGGER.info("success to update business info");
+ LOGGER.info("success to update business info for groupId={}", groupId);
return true;
}
@@ -288,7 +289,7 @@ public class DataStreamServiceImpl implements DataStreamService {
LOGGER.debug("begin to delete data stream field, streamId={}", streamId);
streamFieldMapper.logicDeleteAllByIdentifier(groupId, streamId);
- LOGGER.info("success to delete data stream, ext property and fields");
+ LOGGER.info("success to delete data stream, ext property and fields for groupId={}", groupId);
return true;
}
@@ -341,10 +342,10 @@ public class DataStreamServiceImpl implements DataStreamService {
*/
private boolean hasDataSource(String groupId, String streamId, String dataSourceType) {
boolean exist;
- if (BizConstant.DATA_SOURCE_TYPE_FILE.equalsIgnoreCase(dataSourceType)) {
+ if (BizConstant.DATA_SOURCE_FILE.equalsIgnoreCase(dataSourceType)) {
List<SourceFileDetailInfo> fileDetailList = sourceFileService.listDetailByIdentifier(groupId, streamId);
exist = CollectionUtils.isNotEmpty(fileDetailList);
- } else if (BizConstant.DATA_SOURCE_TYPE_DB.equalsIgnoreCase(dataSourceType)) {
+ } else if (BizConstant.DATA_SOURCE_DB.equalsIgnoreCase(dataSourceType)) {
List<SourceDbDetailInfo> dbDetailList = sourceDbService.listDetailByIdentifier(groupId, streamId);
exist = CollectionUtils.isNotEmpty(dbDetailList);
} else {
@@ -369,7 +370,7 @@ public class DataStreamServiceImpl implements DataStreamService {
summaryInfo.setStorageList(storageList);
}
- LOGGER.info("success to get data stream summary list");
+ LOGGER.info("success to get data stream summary list for groupId={}", groupId);
return summaryInfoList;
}
@@ -500,21 +501,21 @@ public class DataStreamServiceImpl implements DataStreamService {
continue;
}
switch (dataSourceType.toUpperCase(Locale.ROOT)) {
- case BizConstant.DATA_SOURCE_TYPE_FILE:
+ case BizConstant.DATA_SOURCE_FILE:
SourceFileBasicInfo fileBasicInfo = sourceFileService.getBasicByIdentifier(groupId, streamId);
pageInfo.setFileBasicInfo(fileBasicInfo);
List<SourceFileDetailInfo> fileDetailInfoList = sourceFileService.listDetailByIdentifier(groupId,
streamId);
pageInfo.setFileDetailInfoList(fileDetailInfoList);
break;
- case BizConstant.DATA_SOURCE_TYPE_DB:
+ case BizConstant.DATA_SOURCE_DB:
SourceDbBasicInfo dbBasicInfo = sourceDbService.getBasicByIdentifier(groupId, streamId);
pageInfo.setDbBasicInfo(dbBasicInfo);
List<SourceDbDetailInfo> dbDetailInfoList = sourceDbService.listDetailByIdentifier(groupId,
streamId);
pageInfo.setDbDetailInfoList(dbDetailInfoList);
break;
- case BizConstant.DATA_SOURCE_TYPE_AUTO_PUSH:
+ case BizConstant.DATA_SOURCE_AUTO_PUSH:
break;
default:
throw new BusinessException(BizErrorCodeEnum.DATA_SOURCE_TYPE_NOT_SUPPORTED);
@@ -576,6 +577,16 @@ public class DataStreamServiceImpl implements DataStreamService {
}
@Override
+ public List<DataStreamTopicVO> getTopicList(String groupId) {
+ LOGGER.debug("begin bo get topic list by group id={}", groupId);
+ Preconditions.checkNotNull(groupId, BizConstant.GROUP_ID_IS_EMPTY);
+
+ List<DataStreamTopicVO> topicList = streamMapper.selectTopicList(groupId);
+ LOGGER.debug("success to get topic list by groupId={}", groupId);
+ return topicList;
+ }
+
+ @Override
public boolean updateAfterApprove(List<DataStreamApproveInfo> streamApproveList, String operator) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("begin to update stream after approve={}", streamApproveList);
@@ -584,21 +595,23 @@ public class DataStreamServiceImpl implements DataStreamService {
return true;
}
+ String groupId = null;
for (DataStreamApproveInfo info : streamApproveList) {
// Modify mqResourceObj
DataStreamEntity streamEntity = new DataStreamEntity();
- streamEntity.setInlongGroupId(info.getInlongGroupId());
+ groupId = info.getInlongGroupId(); // these groupIds are all the same
+ streamEntity.setInlongGroupId(groupId);
streamEntity.setInlongStreamId(info.getInlongStreamId());
+ // Update status to [DATA_STREAM_CONFIG_ING]
streamEntity.setStatus(EntityStatus.DATA_STREAM_CONFIG_ING.getCode());
streamMapper.updateByIdentifierSelective(streamEntity);
- // Update status to [DATA_STREAM_CONFIG_ING]
// If you need to change data stream info after approve, just do in here
// Modify the storage information
storageService.updateAfterApprove(info.getStorageList(), operator);
}
- LOGGER.info("success to update stream after approve");
+ LOGGER.info("success to update stream after approve for groupId={}", groupId);
return true;
}
@@ -609,7 +622,7 @@ public class DataStreamServiceImpl implements DataStreamService {
// businessMapper.updateStatusByIdentifier(groupId, status, operator);
streamMapper.updateStatusByIdentifier(groupId, streamId, status, operator);
- LOGGER.info("success to update stream after approve");
+ LOGGER.info("success to update stream after approve for groupId={}", groupId);
return true;
}
@@ -627,7 +640,7 @@ public class DataStreamServiceImpl implements DataStreamService {
try {
streamExtMapper.deleteAllByIdentifier(groupId, streamId);
saveExt(groupId, streamId, extInfoList, new Date());
- LOGGER.info("success to update data stream ext");
+ LOGGER.info("success to update data stream ext for groupId={}", groupId);
} catch (Exception e) {
LOGGER.error("failed to update data stream ext: ", e);
throw new BusinessException(BizErrorCodeEnum.DATA_STREAM_EXT_SAVE_FAILED);
@@ -659,7 +672,7 @@ public class DataStreamServiceImpl implements DataStreamService {
try {
streamFieldMapper.deleteAllByIdentifier(groupId, streamId);
saveField(groupId, streamId, fieldInfoList);
- LOGGER.info("success to update data stream field");
+ LOGGER.info("success to update data stream field for groupId={}", groupId);
} catch (Exception e) {
LOGGER.error("failed to update data stream field: ", e);
throw new BusinessException(BizErrorCodeEnum.DATA_STREAM_FIELD_SAVE_FAILED);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageHiveOperation.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageHiveOperation.java
index fa3e541..0d5a268 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageHiveOperation.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageHiveOperation.java
@@ -104,7 +104,7 @@ public class StorageHiveOperation extends StorageBaseOperation {
// Save field information
this.saveHiveFieldOpt(id, hiveInfo.getHiveFieldList());
// Save extended information
- String storageType = BizConstant.STORAGE_TYPE_HIVE;
+ String storageType = BizConstant.STORAGE_HIVE;
this.saveExtOpt(storageType, id, hiveInfo.getExtList());
return id;
@@ -127,8 +127,8 @@ public class StorageHiveOperation extends StorageBaseOperation {
String storageType;
List<StorageExtEntity> extEntities;
- storageType = BizConstant.STORAGE_TYPE_HIVE;
- extEntities = storageExtMapper.selectByStorageTypeAndId(BizConstant.STORAGE_TYPE_HIVE, storageId);
+ storageType = BizConstant.STORAGE_HIVE;
+ extEntities = storageExtMapper.selectByStorageTypeAndId(BizConstant.STORAGE_HIVE, storageId);
List<StorageHiveFieldEntity> fieldEntityList = hiveFieldMapper.selectByStorageId(storageId);
List<StorageHiveFieldInfo> fieldInfoList = CommonBeanUtils
@@ -232,7 +232,7 @@ public class StorageHiveOperation extends StorageBaseOperation {
}
StorageHiveInfo hiveInfo = CommonBeanUtils.copyProperties(entity, StorageHiveInfo::new);
- String storageType = BizConstant.STORAGE_TYPE_HIVE;
+ String storageType = BizConstant.STORAGE_HIVE;
List<StorageExtEntity> extEntityList = storageExtMapper.selectByStorageTypeAndId(storageType, id);
List<StorageExtInfo> extInfoList = CommonBeanUtils.copyListProperties(extEntityList, StorageExtInfo::new);
hiveInfo.setExtList(extInfoList);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageServiceImpl.java
index 0052b6c..82868cf 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/StorageServiceImpl.java
@@ -78,7 +78,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
Preconditions.checkNotNull(storageType, "storageType is empty");
int id;
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
id = hiveOperation.saveHiveStorage(storageInfo, operator);
} else {
LOGGER.error("the storageType={} not support", storageType);
@@ -102,7 +102,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
Preconditions.checkNotNull(storageType, "storageType is empty");
BaseStorageInfo storageInfo;
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
storageInfo = hiveOperation.getHiveStorage(id);
} else {
LOGGER.error("the storageType={} not support", storageType);
@@ -165,7 +165,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
Preconditions.checkNotNull(storageType, "storageType is empty");
PageInfo<? extends BaseStorageListVO> page;
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
page = hiveOperation.getHiveStorageList(request);
} else {
LOGGER.error("the storageType={} not support", storageType);
@@ -195,7 +195,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
String storageType = storageInfo.getStorageType();
Preconditions.checkNotNull(storageType, "storageType is empty");
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
hiveOperation.updateHiveStorage(businessEntity.getStatus(), storageInfo, operator);
} else {
LOGGER.error("the storageType={} not support", storageType);
@@ -219,7 +219,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
Preconditions.checkNotNull(storageType, "storageType is empty");
boolean result;
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
result = hiveOperation.logicDeleteHiveStorage(id, operator);
} else {
LOGGER.error("the storageType={} not support", storageType);
@@ -280,7 +280,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
return resultList;
}
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
resultList = hiveStorageMapper.selectDataStreamExists(groupId, streamIdList);
} else {
LOGGER.error("the storageType={} not support", storageType);
@@ -301,7 +301,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
}
if (hiveStorageMapper.selectCountByIdentifier(groupId, streamId) > 0) {
- resultList.add(BizConstant.STORAGE_TYPE_HIVE);
+ resultList.add(BizConstant.STORAGE_HIVE);
}
LOGGER.info("success to get storage type list");
@@ -322,7 +322,7 @@ public class StorageServiceImpl extends StorageBaseOperation implements StorageS
String storageType = info.getStorageType();
Preconditions.checkNotNull(storageType, "storageType is empty");
- if (BizConstant.STORAGE_TYPE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
+ if (BizConstant.STORAGE_HIVE.equals(storageType.toUpperCase(Locale.ROOT))) {
StorageHiveEntity hiveEntity = new StorageHiveEntity();
hiveEntity.setId(info.getId());
hiveEntity.setModifier(operator);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeConsumeGroupTaskEventListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeConsumerGroupTaskListener.java
similarity index 92%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeConsumeGroupTaskEventListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeConsumerGroupTaskListener.java
index 22875d4..580c249 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeConsumeGroupTaskEventListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeConsumerGroupTaskListener.java
@@ -19,7 +19,7 @@ package org.apache.inlong.manager.service.thirdpart.mq;
import java.util.Collections;
import lombok.extern.slf4j.Slf4j;
-import org.apache.inlong.manager.common.beans.TryBean;
+import org.apache.inlong.manager.common.beans.ReTryConfigBean;
import org.apache.inlong.manager.common.pojo.business.BusinessInfo;
import org.apache.inlong.manager.common.pojo.tubemq.AddTubeConsumeGroupRequest;
import org.apache.inlong.manager.common.pojo.tubemq.AddTubeConsumeGroupRequest.GroupNameJsonSetBean;
@@ -38,7 +38,7 @@ import org.springframework.stereotype.Component;
@Component
@Slf4j
-public class CreateTubeConsumeGroupTaskEventListener implements TaskEventListener {
+public class CreateTubeConsumerGroupTaskListener implements TaskEventListener {
@Autowired
BusinessService businessService;
@@ -53,7 +53,7 @@ public class CreateTubeConsumeGroupTaskEventListener implements TaskEventListene
Integer clusterId;
@Autowired
- TryBean tryBean;
+ ReTryConfigBean reTryConfigBean;
@Override
public TaskEvent event() {
@@ -77,13 +77,13 @@ public class CreateTubeConsumeGroupTaskEventListener implements TaskEventListene
// Query whether the tube topic exists
boolean topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
- Integer tryNumber = tryBean.getMaxAttempts();
- Long delay = tryBean.getDelay();
+ Integer tryNumber = reTryConfigBean.getMaxAttempts();
+ Long delay = reTryConfigBean.getDelay();
while (!topicExist && --tryNumber > 0) {
log.info("check whether the tube topic exists, try count={}", tryNumber);
try {
Thread.sleep(delay);
- delay *= tryBean.getMultiplier();
+ delay *= reTryConfigBean.getMultiplier();
topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
} catch (InterruptedException e) {
log.error("check the tube topic exists error", e);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeTopicTaskEventListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeTopicTaskListener.java
similarity index 97%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeTopicTaskEventListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeTopicTaskListener.java
index 2f9c24c..01d73c6 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeTopicTaskEventListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/mq/CreateTubeTopicTaskListener.java
@@ -36,7 +36,7 @@ import org.springframework.stereotype.Component;
*/
@Component
@Slf4j
-public class CreateTubeTopicTaskEventListener implements TaskEventListener {
+public class CreateTubeTopicTaskListener implements TaskEventListener {
@Autowired
private TubeMqOptService tubeMqOptService;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/sort/PushHiveConfigToSortEventListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/sort/PushHiveConfigTaskListener.java
similarity index 98%
rename from inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/sort/PushHiveConfigToSortEventListener.java
rename to inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/sort/PushHiveConfigTaskListener.java
index 9026453..d27ef01 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/sort/PushHiveConfigToSortEventListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/thirdpart/sort/PushHiveConfigTaskListener.java
@@ -51,7 +51,7 @@ import org.springframework.stereotype.Component;
@Slf4j
@Component
-public class PushHiveConfigToSortEventListener implements TaskEventListener {
+public class PushHiveConfigTaskListener implements TaskEventListener {
@Autowired
private StorageService storageService;
@@ -83,7 +83,7 @@ public class PushHiveConfigToSortEventListener implements TaskEventListener {
for (StorageHiveEntity hiveEntity : storageHiveEntities) {
Integer storageId = hiveEntity.getId();
StorageHiveInfo hiveStorage = (StorageHiveInfo) storageService
- .getById(BizConstant.STORAGE_TYPE_HIVE, storageId);
+ .getById(BizConstant.STORAGE_HIVE, storageId);
if (log.isDebugEnabled()) {
log.debug("hive storage info: {}", hiveStorage);
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newbusiness/CreateResourceWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newbusiness/CreateResourceWorkflowDefinition.java
index 50a6afa..f3221db 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newbusiness/CreateResourceWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newbusiness/CreateResourceWorkflowDefinition.java
@@ -27,9 +27,9 @@ import org.apache.inlong.manager.dao.entity.DataStreamEntity;
import org.apache.inlong.manager.dao.mapper.DataStreamEntityMapper;
import org.apache.inlong.manager.service.core.StorageService;
import org.apache.inlong.manager.service.thirdpart.hive.CreateHiveTableForAllStreamListener;
-import org.apache.inlong.manager.service.thirdpart.mq.CreateTubeConsumeGroupTaskEventListener;
-import org.apache.inlong.manager.service.thirdpart.mq.CreateTubeTopicTaskEventListener;
-import org.apache.inlong.manager.service.thirdpart.sort.PushHiveConfigToSortEventListener;
+import org.apache.inlong.manager.service.thirdpart.mq.CreateTubeConsumerGroupTaskListener;
+import org.apache.inlong.manager.service.thirdpart.mq.CreateTubeTopicTaskListener;
+import org.apache.inlong.manager.service.thirdpart.sort.PushHiveConfigTaskListener;
import org.apache.inlong.manager.service.workflow.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
import org.apache.inlong.manager.service.workflow.newbusiness.listener.CreateResourceCompleteProcessListener;
@@ -59,16 +59,16 @@ public class CreateResourceWorkflowDefinition implements WorkflowDefinition {
private CreateResourceFailedProcessListener createResourceFailedProcessListener;
@Autowired
- private CreateTubeTopicTaskEventListener createTubeTopicTaskEventListener;
+ private CreateTubeTopicTaskListener createTubeTopicTaskListener;
@Autowired
- private CreateTubeConsumeGroupTaskEventListener createTubeConsumeGroupTaskEventListener;
+ private CreateTubeConsumerGroupTaskListener createTubeConsumerGroupTaskListener;
@Autowired
private CreateHiveTableForAllStreamListener createHiveTableForAllStreamListener;
@Autowired
- private PushHiveConfigToSortEventListener pushHiveConfigToSortEventListener;
+ private PushHiveConfigTaskListener pushHiveConfigTaskListener;
@Autowired
private StorageService storageService;
@@ -104,7 +104,7 @@ public class CreateResourceWorkflowDefinition implements WorkflowDefinition {
createTubeTopicTask.setSkipResolver(c -> {
CreateResourceWorkflowForm form = (CreateResourceWorkflowForm) c.getProcessForm();
BusinessInfo businessInfo = form.getBusinessInfo();
- if (BizConstant.MIDDLEWARE_TYPE_TUBE.equalsIgnoreCase(businessInfo.getMiddlewareType())) {
+ if (BizConstant.MIDDLEWARE_TUBE.equalsIgnoreCase(businessInfo.getMiddlewareType())) {
return false;
}
log.warn("not need to create tube resource for groupId={}, as the middleware type is {}",
@@ -113,14 +113,14 @@ public class CreateResourceWorkflowDefinition implements WorkflowDefinition {
});
createTubeTopicTask.setName("createTubeTopic");
createTubeTopicTask.setDisplayName("Create Tube Topic");
- createTubeTopicTask.addListener(createTubeTopicTaskEventListener);
+ createTubeTopicTask.addListener(createTubeTopicTaskListener);
process.addTask(createTubeTopicTask);
ServiceTask createConsumerGroupForSortTask = new ServiceTask();
createConsumerGroupForSortTask.setSkipResolver(c -> {
CreateResourceWorkflowForm form = (CreateResourceWorkflowForm) c.getProcessForm();
BusinessInfo businessInfo = form.getBusinessInfo();
- if (BizConstant.MIDDLEWARE_TYPE_TUBE.equalsIgnoreCase(businessInfo.getMiddlewareType())) {
+ if (BizConstant.MIDDLEWARE_TUBE.equalsIgnoreCase(businessInfo.getMiddlewareType())) {
return false;
}
log.warn("no need to create tube resource for groupId={}, as the middleware type is {}",
@@ -129,14 +129,14 @@ public class CreateResourceWorkflowDefinition implements WorkflowDefinition {
});
createConsumerGroupForSortTask.setName("createConsumerGroupForSort");
createConsumerGroupForSortTask.setDisplayName("Create Consumer Group For Sort");
- createConsumerGroupForSortTask.addListener(createTubeConsumeGroupTaskEventListener);
+ createConsumerGroupForSortTask.addListener(createTubeConsumerGroupTaskListener);
process.addTask(createConsumerGroupForSortTask);
ServiceTask createHiveTablesTask = new ServiceTask();
createHiveTablesTask.setSkipResolver(c -> {
CreateResourceWorkflowForm form = (CreateResourceWorkflowForm) c.getProcessForm();
List<String> dsForHive = storageService
- .filterStreamIdByStorageType(form.getInlongGroupId(), BizConstant.STORAGE_TYPE_HIVE,
+ .filterStreamIdByStorageType(form.getInlongGroupId(), BizConstant.STORAGE_HIVE,
streamMapper
.selectByGroupId(form.getInlongGroupId())
.stream()
@@ -157,7 +157,7 @@ public class CreateResourceWorkflowDefinition implements WorkflowDefinition {
ServiceTask pushSortConfig = new ServiceTask();
pushSortConfig.setName("pushSortConfig");
pushSortConfig.setDisplayName("Push Sort Config");
- pushSortConfig.addListener(pushHiveConfigToSortEventListener);
+ pushSortConfig.addListener(pushHiveConfigTaskListener);
process.addTask(pushSortConfig);
startEvent.addNext(createTubeTopicTask);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newconsumption/listener/ConsumptionCompleteProcessListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newconsumption/listener/ConsumptionCompleteProcessListener.java
index 619e5cf..605b8c1 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newconsumption/listener/ConsumptionCompleteProcessListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newconsumption/listener/ConsumptionCompleteProcessListener.java
@@ -82,7 +82,7 @@ public class ConsumptionCompleteProcessListener implements ProcessEventListener
String middlewareType = workflowForm.getConsumptionInfo().getMiddlewareType();
- if (BizConstant.MIDDLEWARE_TYPE_TUBE.equalsIgnoreCase(middlewareType)) {
+ if (BizConstant.MIDDLEWARE_TUBE.equalsIgnoreCase(middlewareType)) {
createTubeConsumerGroup(workflowForm.getConsumptionInfo());
return ListenerResult.success("Create Tube Consumer Group");
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newstream/SingleStreamWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newstream/SingleStreamWorkflowDefinition.java
index 8e904e6..3b16958 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newstream/SingleStreamWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/newstream/SingleStreamWorkflowDefinition.java
@@ -24,7 +24,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.inlong.manager.common.enums.BizConstant;
import org.apache.inlong.manager.service.core.StorageService;
import org.apache.inlong.manager.service.thirdpart.hive.CreateHiveTableForOneStreamListener;
-import org.apache.inlong.manager.service.thirdpart.sort.PushHiveConfigToSortEventListener;
+import org.apache.inlong.manager.service.thirdpart.sort.PushHiveConfigTaskListener;
import org.apache.inlong.manager.service.workflow.ProcessName;
import org.apache.inlong.manager.service.workflow.WorkflowDefinition;
import org.apache.inlong.manager.service.workflow.newbusiness.CreateResourceWorkflowForm;
@@ -54,7 +54,7 @@ public class SingleStreamWorkflowDefinition implements WorkflowDefinition {
@Autowired
private CreateHiveTableForOneStreamListener createHiveTableForOneStreamListener;
@Autowired
- private PushHiveConfigToSortEventListener pushHiveConfigToSortEventListener;
+ private PushHiveConfigTaskListener pushHiveConfigTaskListener;
@Override
public Process defineProcess() {
@@ -84,7 +84,7 @@ public class SingleStreamWorkflowDefinition implements WorkflowDefinition {
CreateResourceWorkflowForm form = (CreateResourceWorkflowForm) c.getProcessForm();
String groupId = form.getInlongGroupId();
String streamId = form.getInlongStreamId();
- List<String> dsForHive = storageService.filterStreamIdByStorageType(groupId, BizConstant.STORAGE_TYPE_HIVE,
+ List<String> dsForHive = storageService.filterStreamIdByStorageType(groupId, BizConstant.STORAGE_HIVE,
Collections.singletonList(streamId));
if (CollectionUtils.isEmpty(dsForHive)) {
log.warn("business [{}] adn data stream [{}] does not have storage, skip create hive table", groupId,
@@ -102,7 +102,7 @@ public class SingleStreamWorkflowDefinition implements WorkflowDefinition {
ServiceTask pushSortConfig = new ServiceTask();
pushSortConfig.setName("pushSortConfig");
pushSortConfig.setDisplayName("Push Sort Configuration");
- pushSortConfig.addListener(pushHiveConfigToSortEventListener);
+ pushSortConfig.addListener(pushHiveConfigTaskListener);
process.addTask(pushSortConfig);
startEvent.addNext(createHiveTableTask);
diff --git a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/ConsumptionController.java b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/ConsumptionController.java
index bce3a74..11cd284 100644
--- a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/ConsumptionController.java
+++ b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/ConsumptionController.java
@@ -54,28 +54,28 @@ public class ConsumptionController {
@Autowired
private ConsumptionService consumptionService;
- @GetMapping("summary")
+ @GetMapping("/summary")
@ApiOperation(value = "Get data consumption summary")
public Response<ConsumptionSummary> getSummary(ConsumptionQuery query) {
query.setUserName(LoginUserUtil.getLoginUserDetail().getUserName());
return Response.success(consumptionService.getSummary(query));
}
- @GetMapping("list")
+ @GetMapping("/list")
@ApiOperation(value = "List data consumptions")
public Response<PageInfo<ConsumptionListVo>> list(ConsumptionQuery query) {
query.setUserName(LoginUserUtil.getLoginUserDetail().getUserName());
return Response.success(consumptionService.list(query));
}
- @GetMapping("get/{id}")
+ @GetMapping("/get/{id}")
@ApiOperation(value = "Get consumption details")
public Response<ConsumptionInfo> getDetail(
@ApiParam(value = "Consumption ID", required = true) @PathVariable(name = "id") Integer id) {
return Response.success(consumptionService.getInfo(id));
}
- @DeleteMapping("delete/{id}")
+ @DeleteMapping("/delete/{id}")
@OperationLog(operation = OperationType.DELETE)
@ApiOperation(value = "Delete data consumption")
public Response<Object> delete(
@@ -84,7 +84,7 @@ public class ConsumptionController {
return Response.success();
}
- @PostMapping("save")
+ @PostMapping("/save")
@OperationLog(operation = OperationType.UPDATE)
@ApiOperation(value = "Save data consumption", notes = "Full coverage")
public Response<Integer> saveConsumptionInfo(
@@ -93,7 +93,7 @@ public class ConsumptionController {
return Response.success(consumptionService.save(consumptionInfo, currentUser));
}
- @PostMapping("update/{id}")
+ @PostMapping("/update/{id}")
@OperationLog(operation = OperationType.UPDATE)
@ApiOperation(value = "Update data consumption")
public Response<Integer> updateConsumptionInfo(
@@ -104,7 +104,7 @@ public class ConsumptionController {
return Response.success(consumptionService.update(consumptionUpdateInfo, currentUser));
}
- @PostMapping("startProcess/{id}")
+ @PostMapping("/startProcess/{id}")
@OperationLog(operation = OperationType.UPDATE)
@ApiOperation(value = "Start approval process")
@ApiImplicitParam(name = "id", value = "Consumption ID", dataTypeClass = Integer.class, required = true)
diff --git a/inlong-manager/manager-web/src/main/resources/application-dev.properties b/inlong-manager/manager-web/src/main/resources/application-dev.properties
index 1d47a34..16a5a30 100644
--- a/inlong-manager/manager-web/src/main/resources/application-dev.properties
+++ b/inlong-manager/manager-web/src/main/resources/application-dev.properties
@@ -59,3 +59,8 @@ cluster.zk.root=inlong_hive
# Application name in Sort
sort.appName=inlong_app
+
+# Pulsar admin URL
+pulsar.adminUrl=http://127.0.0.1:8080,127.0.0.2:8080,127.0.0.3:8080
+# Pulsar master address
+pulsar.serviceUrl=pulsar://127.0.0.1:6650,127.0.0.1:6650,127.0.0.1:6650
diff --git a/inlong-manager/manager-web/src/main/resources/application-prod.properties b/inlong-manager/manager-web/src/main/resources/application-prod.properties
index 1985075..5da7518 100644
--- a/inlong-manager/manager-web/src/main/resources/application-prod.properties
+++ b/inlong-manager/manager-web/src/main/resources/application-prod.properties
@@ -58,3 +58,8 @@ cluster.zk.root=inlong_hive
# Application name in Sort
sort.appName=inlong_app
+
+# Pulsar admin URL
+pulsar.adminUrl=http://127.0.0.1:8080,127.0.0.2:8080,127.0.0.3:8080
+# Pulsar master address
+pulsar.serviceUrl=pulsar://127.0.0.1:6650,127.0.0.1:6650,127.0.0.1:6650
diff --git a/inlong-manager/manager-web/src/main/resources/application-test.properties b/inlong-manager/manager-web/src/main/resources/application-test.properties
index 90d1ac4..16a5a30 100644
--- a/inlong-manager/manager-web/src/main/resources/application-test.properties
+++ b/inlong-manager/manager-web/src/main/resources/application-test.properties
@@ -60,10 +60,7 @@ cluster.zk.root=inlong_hive
# Application name in Sort
sort.appName=inlong_app
-# Configure http client
-#common.http-client.maxTotal=5001
-#common.http-client.defaultMaxPerRoute=2001
-#common.http-client.validateAfterInactivity=5001
-#common.http-client.connectionTimeout=3001
-#common.http-client.readTimeout=10001
-#common.http-client.connectionRequestTimeout=3001
\ No newline at end of file
+# Pulsar admin URL
+pulsar.adminUrl=http://127.0.0.1:8080,127.0.0.2:8080,127.0.0.3:8080
+# Pulsar master address
+pulsar.serviceUrl=pulsar://127.0.0.1:6650,127.0.0.1:6650,127.0.0.1:6650
diff --git a/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/service/core/BusinessServiceTest.java b/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/service/core/BusinessServiceTest.java
new file mode 100644
index 0000000..f085c61
--- /dev/null
+++ b/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/service/core/BusinessServiceTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.core;
+
+import org.apache.inlong.manager.common.enums.BizConstant;
+import org.apache.inlong.manager.common.pojo.business.BusinessInfo;
+import org.apache.inlong.manager.common.pojo.business.BusinessPulsarInfo;
+import org.apache.inlong.manager.web.ServiceBaseTest;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Business Service Test
+ */
+public class BusinessServiceTest extends ServiceBaseTest {
+
+ @Autowired
+ private BusinessService businessService;
+
+ public String saveBusiness(String operator) {
+ String groupName = "test_group";
+ BusinessInfo businessInfo = new BusinessInfo();
+ businessInfo.setName(groupName);
+ businessInfo.setMiddlewareType(BizConstant.MIDDLEWARE_PULSAR);
+ businessInfo.setCreator(operator);
+
+ BusinessPulsarInfo pulsarInfo = new BusinessPulsarInfo();
+ pulsarInfo.setMiddlewareType(BizConstant.MIDDLEWARE_PULSAR);
+ pulsarInfo.setEnsemble(3);
+ pulsarInfo.setWriteQuorum(3);
+ pulsarInfo.setAckQuorum(2);
+
+ businessInfo.setMqExtInfo(pulsarInfo);
+
+ return businessService.save(businessInfo, operator);
+ }
+
+ @Test
+ public void testSave() {
+ String operator = "test_user";
+ String groupId = this.saveBusiness(operator);
+ Assert.assertNotNull(groupId);
+ }
+
+ @Test
+ public void testDelete() {
+ String operator = "test_user";
+ String groupId = this.saveBusiness(operator);
+ boolean result = businessService.delete(groupId, operator);
+ Assert.assertTrue(result);
+ }
+
+}
diff --git a/inlong-manager/manager-web/src/test/resources/application-test.properties b/inlong-manager/manager-web/src/test/resources/application-test.properties
index 167a2ec..6333bd0 100644
--- a/inlong-manager/manager-web/src/test/resources/application-test.properties
+++ b/inlong-manager/manager-web/src/test/resources/application-test.properties
@@ -62,3 +62,8 @@ cluster.zk.root=inlong_hive
# Application name in Sort
sort.appName=inlong_app
+
+# Pulsar admin URL
+pulsar.adminUrl=http://127.0.0.1:8080,127.0.0.2:8080,127.0.0.3:8080
+# Pulsar master address
+pulsar.serviceUrl=pulsar://127.0.0.1:6650,127.0.0.1:6650,127.0.0.1:6650
diff --git a/inlong-manager/manager-web/src/test/resources/sql/apache_inlong_manager.sql b/inlong-manager/manager-web/src/test/resources/sql/apache_inlong_manager.sql
index 22ffb4a..9b8e124 100644
--- a/inlong-manager/manager-web/src/test/resources/sql/apache_inlong_manager.sql
+++ b/inlong-manager/manager-web/src/test/resources/sql/apache_inlong_manager.sql
@@ -27,7 +27,7 @@ CREATE TABLE `agent_heartbeat_log`
`ip` varchar(64) NOT NULL COMMENT 'agent host ip',
`version` varchar(128) DEFAULT NULL,
`heartbeat_msg` text DEFAULT NULL COMMENT 'massage in heartbeat request',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`ip`)
);
@@ -58,7 +58,7 @@ CREATE TABLE `agent_sys_conf`
`stat_interval_sec` int(11) NOT NULL DEFAULT '60' COMMENT 'Statistical message sending frequency',
`conf_refresh_interval_secs` int(11) NOT NULL DEFAULT '300' COMMENT 'The frequency at which the Agent regularly pulls the configuration from the InLongManager',
`flow_size` int(11) DEFAULT '1048576000',
- `bufferSize` int(11) DEFAULT NULL COMMENT 'bufferSize, default 1048576',
+ `bufferSize` int(11) DEFAULT '1048576' COMMENT 'bufferSize, default 1048576',
`compress` tinyint(2) DEFAULT NULL COMMENT 'Whether to compress',
`event_check_interval` int(11) DEFAULT NULL COMMENT 'File scanning period',
`is_calMD5` tinyint(2) DEFAULT NULL COMMENT 'Do you want to calculate the cumulative md5 of read characters',
@@ -71,43 +71,68 @@ CREATE TABLE `agent_sys_conf`
DROP TABLE IF EXISTS `business`;
CREATE TABLE `business`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
- `name` varchar(128) DEFAULT '' COMMENT 'Business name, English, numbers and underscore',
- `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'Business Introduction',
- `middleware_type` varchar(10) DEFAULT 'Tube' COMMENT 'The middleware type of data storage, high throughput: Tube',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, for Tube, its Topic',
- `daily_records` int(11) DEFAULT NULL COMMENT 'Number of access records per day, unit: 10,000 records per day',
- `daily_storage` int(11) DEFAULT NULL COMMENT 'Access size by day, unit: GB per day',
- `peak_records` int(11) DEFAULT NULL COMMENT 'Access peak per second, unit: records per second',
- `max_length` int(11) DEFAULT NULL COMMENT 'The maximum length of a single piece of data, unit: Byte',
- `schema_name` varchar(128) DEFAULT NULL COMMENT 'Data type, associated data_schema table',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
- `status` int(11) DEFAULT '21' COMMENT 'Business status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
+ `name` varchar(128) DEFAULT '' COMMENT 'Business name, English, numbers and underscore',
+ `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
+ `description` varchar(256) DEFAULT '' COMMENT 'Business Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `mq_resource_obj` varchar(128) NOT NULL COMMENT 'MQ resource object, for Tube, its Topic, for Pulsar, its Namespace',
+ `daily_records` int(11) DEFAULT '10' COMMENT 'Number of access records per day, unit: 10,000 records per day',
+ `daily_storage` int(11) DEFAULT '10' COMMENT 'Access size by day, unit: GB per day',
+ `peak_records` int(11) DEFAULT '1000' COMMENT 'Access peak per second, unit: records per second',
+ `max_length` int(11) DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte',
+ `schema_name` varchar(128) DEFAULT NULL COMMENT 'Data type, associated data_schema table',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
+ `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
+ `status` int(11) DEFAULT '21' COMMENT 'Business status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_business` (`inlong_group_id`, `is_deleted`, `modify_time`)
);
+
+-- ----------------------------
+-- Table structure for business_pulsar
+-- ----------------------------
+DROP TABLE IF EXISTS `business_pulsar`;
+CREATE TABLE `business_pulsar`
+(
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
+ `ensemble` int(3) DEFAULT '3' COMMENT 'The writable nodes number of ledger',
+ `write_quorum` int(3) DEFAULT '3' COMMENT 'The copies number of ledger',
+ `ack_quorum` int(3) DEFAULT '2' COMMENT 'The number of requested acks',
+ `retention_time` int(11) DEFAULT '72' COMMENT 'Message storage time',
+ `retention_time_unit` char(20) DEFAULT 'hours' COMMENT 'The unit of the message storage time',
+ `ttl` int(11) DEFAULT '24' COMMENT 'Message time-to-live duration',
+ `ttl_unit` varchar(20) DEFAULT 'hours' COMMENT 'The unit of time-to-live duration',
+ `retention_size` int(11) DEFAULT '-1' COMMENT 'Message size',
+ `retention_size_unit` varchar(20) DEFAULT 'MB' COMMENT 'The unit of message size',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ PRIMARY KEY (`id`)
+);
+
-- ----------------------------
-- Table structure for business_ext
-- ----------------------------
DROP TABLE IF EXISTS `business_ext`;
CREATE TABLE `business_ext`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id',
- `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- PRIMARY KEY (`id`)
+ `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
+ `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ PRIMARY KEY (`id`),
+ KEY `index_group_id` (`inlong_group_id`)
);
-- ----------------------------
@@ -118,19 +143,19 @@ CREATE TABLE `cluster_info`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`name` varchar(128) NOT NULL COMMENT 'cluster name',
- `type` varchar(32) DEFAULT NULL COMMENT 'Cluster type, including TUBE, ZOOKEEPER, etc.',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Cluster IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Cluster port number',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `type` varchar(32) NOT NULL COMMENT 'Cluster type, including TUBE, PULSAR, etc.',
+ `ip` varchar(64) NOT NULL COMMENT 'Cluster IP address',
+ `port` int(11) NOT NULL COMMENT 'Cluster port',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Name of responsible person, separated by commas',
`url` varchar(256) DEFAULT NULL COMMENT 'Cluster URL address',
`is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
`ext_props` text DEFAULT NULL COMMENT 'extended properties',
`status` int(11) DEFAULT '1' COMMENT 'cluster status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -140,32 +165,32 @@ CREATE TABLE `cluster_info`
DROP TABLE IF EXISTS `common_db_server`;
CREATE TABLE `common_db_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
- `db_type` varchar(128) DEFAULT NULL COMMENT 'DB type, such as MySQL, Oracle',
- `db_server_ip` varchar(64) DEFAULT NULL COMMENT 'DB Server IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `username` varchar(64) DEFAULT NULL COMMENT 'Username',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
- `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
- `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
- `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
- `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
- `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
- `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
- `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `connection_name` varchar(128) NOT NULL COMMENT 'The name of the database connection',
+ `db_type` varchar(128) DEFAULT 'MySQL' COMMENT 'DB type, such as MySQL, Oracle',
+ `db_server_ip` varchar(64) NOT NULL COMMENT 'DB Server IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
+ `username` varchar(64) NOT NULL COMMENT 'Username',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `has_select` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission select, 0: No, 1: Yes',
+ `has_insert` tinyint(1) DEFAULT '0' COMMENT 'Is there DB permission to insert, 0: No, 1: Yes',
+ `has_update` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission update, 0: No, 1: Yes',
+ `has_delete` tinyint(1) DEFAULT '0' COMMENT 'Is there a DB permission to delete, 0: No, 1: Yes',
+ `in_charges` varchar(512) NOT NULL COMMENT 'DB person in charge, separated by a comma when there are multiple ones',
+ `is_region_id` tinyint(1) DEFAULT '0' COMMENT 'Whether it contains a region ID, 0: No, 1: Yes',
+ `db_description` varchar(256) DEFAULT NULL COMMENT 'DB description',
+ `backup_db_server_ip` varchar(64) DEFAULT NULL COMMENT 'Backup DB HOST',
+ `backup_db_port` int(11) DEFAULT NULL COMMENT 'Backup DB port',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
);
@@ -175,22 +200,22 @@ CREATE TABLE `common_db_server`
DROP TABLE IF EXISTS `common_file_server`;
CREATE TABLE `common_file_server`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `ip` varchar(64) DEFAULT NULL COMMENT 'Data source IP',
- `port` int(11) DEFAULT NULL COMMENT 'Port number',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
- `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
- `username` varchar(64) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
- `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy, LoadProxy',
+ `ip` varchar(64) NOT NULL COMMENT 'Data source IP',
+ `port` int(11) NOT NULL COMMENT 'Port number',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: No, 1: Yes',
+ `issue_type` varchar(128) DEFAULT NULL COMMENT 'Issuance method, such as SSH, TCS, etc.',
+ `username` varchar(64) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `visible_person` varchar(1024) DEFAULT NULL COMMENT 'List of visible persons, separated by commas',
+ `visible_group` varchar(1024) DEFAULT NULL COMMENT 'List of visible groups, separated by commas',
PRIMARY KEY (`id`)
);
@@ -200,21 +225,21 @@ CREATE TABLE `common_file_server`
DROP TABLE IF EXISTS `consumption`;
CREATE TABLE `consumption`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `consumer_group_name` varchar(255) NOT NULL COMMENT 'Consumer group name',
- `consumer_group_id` varchar(255) NOT NULL COMMENT 'Consumer group ID',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Person in charge of consumption',
- `inlong_group_id` varchar(255) DEFAULT NULL COMMENT 'Business group id',
- `middleware_type` varchar(64) DEFAULT NULL COMMENT 'The middleware type of data storage, high throughput: Tube',
- `topic` varchar(255) DEFAULT NULL COMMENT 'Consumption topic',
- `filter_enabled` int(2) DEFAULT NULL COMMENT 'Whether to filter',
- `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Consumption data stream ID',
- `status` int(11) NOT NULL COMMENT 'Status: draft, pending approval, approval rejected, approval passed',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
- `is_deleted` int(2) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `consumer_group_name` varchar(255) NOT NULL COMMENT 'consumer group name',
+ `consumer_group_id` varchar(255) NOT NULL COMMENT 'Consumer group ID',
+ `in_charges` varchar(512) NOT NULL COMMENT 'Person in charge of consumption',
+ `inlong_group_id` varchar(255) NOT NULL COMMENT 'Business group id',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `topic` varchar(255) NOT NULL COMMENT 'Consumption topic',
+ `filter_enabled` int(2) DEFAULT '0' COMMENT 'Whether to filter, default 0, not filter consume',
+ `inlong_stream_id` varchar(1024) DEFAULT NULL COMMENT 'Data stream ID for consumption, if filter_enable is 1, it cannot empty',
+ `status` int(11) NOT NULL COMMENT 'Status: draft, pending approval, approval rejected, approval passed',
+ `creator` varchar(64) NOT NULL COMMENT 'creator',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `is_deleted` int(2) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
PRIMARY KEY (`id`)
);
@@ -224,22 +249,22 @@ CREATE TABLE `consumption`
DROP TABLE IF EXISTS `data_proxy_cluster`;
CREATE TABLE `data_proxy_cluster`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `name` varchar(128) DEFAULT NULL COMMENT 'cluster name',
- `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
- `address` varchar(128) DEFAULT NULL COMMENT 'cluster address',
- `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
- `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
- `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `ext_props` text DEFAULT NULL COMMENT 'Extended properties',
- `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
- `create_time` timestamp NULL DEFAULT NULL COMMENT 'Create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `name` varchar(128) NOT NULL COMMENT 'cluster name',
+ `description` varchar(500) DEFAULT NULL COMMENT 'cluster description',
+ `address` varchar(128) NOT NULL COMMENT 'cluster address',
+ `port` varchar(256) DEFAULT '46801' COMMENT 'Access port number, multiple ports are separated by a comma',
+ `is_backup` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a backup cluster, 0: no, 1: yes',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
+ `net_type` varchar(20) DEFAULT NULL COMMENT 'Cluster network type, internal, or public',
+ `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
+ `ext_props` text DEFAULT NULL COMMENT 'Extended properties',
+ `status` int(11) DEFAULT '1' COMMENT 'Cluster status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -275,7 +300,7 @@ CREATE TABLE `data_source_cmd_config`
`specified_data_time` varchar(64) NOT NULL,
`bSend` tinyint(1) NOT NULL,
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Last update time ',
- `create_time` timestamp NULL DEFAULT NULL,
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`result_info` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`)
);
@@ -287,11 +312,11 @@ DROP TABLE IF EXISTS `data_stream`;
CREATE TABLE `data_stream`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id, non-deleted globally unique',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
- `name` varchar(64) DEFAULT NULL COMMENT 'The name of the data stream page display, can be Chinese',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id, non-deleted globally unique',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `name` varchar(64) NOT NULL COMMENT 'The name of the data stream page display, can be Chinese',
`description` varchar(256) DEFAULT '' COMMENT 'Introduction to data stream',
- `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id',
+ `mq_resource_obj` varchar(128) DEFAULT NULL COMMENT 'MQ resource object, in the data stream, Tube is data_stream_id, Pulsar is Topic',
`data_source_type` varchar(32) DEFAULT 'FILE' COMMENT 'Data source type, including: FILE, DB, Auto-Push (DATA_PROXY_SDK, HTTP)',
`storage_period` int(11) DEFAULT '1' COMMENT 'The storage period of data in MQ, unit: day',
`data_type` varchar(20) DEFAULT 'TEXT' COMMENT 'Data type, there are: TEXT, KEY-VALUE, PB, BON, TEXT and BON should be treated differently',
@@ -302,10 +327,10 @@ CREATE TABLE `data_stream`
`status` int(11) DEFAULT '0' COMMENT 'Data stream status',
`previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) DEFAULT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
`temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`)
);
@@ -316,13 +341,13 @@ CREATE TABLE `data_stream`
DROP TABLE IF EXISTS `data_stream_ext`;
CREATE TABLE `data_stream_ext`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
+ `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -332,23 +357,24 @@ CREATE TABLE `data_stream_ext`
DROP TABLE IF EXISTS `data_stream_field`;
CREATE TABLE `data_stream_field`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
- `inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
- `is_predefined_field` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a predefined field, 0: no, 1: yes',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
- `field_value` varchar(128) DEFAULT NULL COMMENT 'Field value, required if it is a predefined field',
- `pre_expression` varchar(256) DEFAULT NULL COMMENT 'Pre-defined field value expression',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
- `field_comment` varchar(50) DEFAULT NULL COMMENT 'Field description',
- `rank_num` smallint(6) DEFAULT '0' COMMENT 'Field order (front-end display field order)',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `is_exist` tinyint(1) DEFAULT '0' COMMENT 'Does it exist, 0: does not exist, 1: exists',
- `bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
- `bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
- `encrypt_level` varchar(20) DEFAULT NULL COMMENT 'Encryption level',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
+ `is_predefined_field` tinyint(1) DEFAULT '0' COMMENT 'Whether it is a predefined field, 0: no, 1: yes',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
+ `field_value` varchar(128) DEFAULT NULL COMMENT 'Field value, required if it is a predefined field',
+ `pre_expression` varchar(256) DEFAULT NULL COMMENT 'Pre-defined field value expression',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
+ `field_comment` varchar(50) DEFAULT NULL COMMENT 'Field description',
+ `rank_num` smallint(6) DEFAULT '0' COMMENT 'Field order (front-end display field order)',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `is_exist` tinyint(1) DEFAULT '0' COMMENT 'Does it exist, 0: does not exist, 1: exists',
+ `bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
+ `bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
+ `encrypt_level` varchar(20) DEFAULT NULL COMMENT 'Encryption level',
PRIMARY KEY (`id`)
);
+
-- ----------------------------
-- Table structure for operation_log
-- ----------------------------
@@ -380,8 +406,8 @@ DROP TABLE IF EXISTS `role`;
CREATE TABLE `role`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
- `role_code` varchar(100) NOT NULL COMMENT 'role code',
- `role_name` varchar(255) NOT NULL COMMENT 'role Chinese name',
+ `role_code` varchar(100) NOT NULL COMMENT 'Role code',
+ `role_name` varchar(255) NOT NULL COMMENT 'Role Chinese name',
`create_time` datetime NOT NULL,
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_by` varchar(255) NOT NULL,
@@ -396,16 +422,16 @@ CREATE TABLE `role`
DROP TABLE IF EXISTS `source_db_basic`;
CREATE TABLE `source_db_basic`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(256) NOT NULL COMMENT 'Owning data stream id',
- `sync_type` tinyint(1) DEFAULT '0' COMMENT 'Data synchronization type, 0: FULL, full amount, 1: INCREMENTAL, incremental',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
+ `sync_type` tinyint(1) DEFAULT '0' COMMENT 'Data synchronization type, 0: FULL, full amount, 1: INCREMENTAL, incremental',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save intermediate data that has not been submitted or approved after modification',
PRIMARY KEY (`id`)
);
@@ -415,25 +441,25 @@ CREATE TABLE `source_db_basic`
DROP TABLE IF EXISTS `source_db_detail`;
CREATE TABLE `source_db_detail`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `access_type` varchar(20) DEFAULT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Database name',
- `transfer_ip` varchar(64) DEFAULT NULL COMMENT 'Transfer IP',
- `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
- `table_name` varchar(128) DEFAULT NULL COMMENT 'Data table name, required for increment',
- `table_fields` longtext COMMENT 'Data table fields, multiple are separated by half-width commas, required for increment',
- `data_sql` longtext COMMENT 'SQL statement to collect source data, required for full amount',
- `crontab` varchar(56) DEFAULT NULL COMMENT 'Timed scheduling expression, required for full amount',
- `status` int(11) DEFAULT '0' COMMENT 'Data source status',
- `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save unsubmitted and unapproved intermediate data after modification',
+ `access_type` varchar(20) NOT NULL COMMENT 'Collection type, with Agent, DataProxy client, LoadProxy',
+ `db_name` varchar(128) DEFAULT NULL COMMENT 'database name',
+ `transfer_ip` varchar(64) DEFAULT NULL COMMENT 'Transfer IP',
+ `connection_name` varchar(128) DEFAULT NULL COMMENT 'The name of the database connection',
+ `table_name` varchar(128) DEFAULT NULL COMMENT 'Data table name, required for increment',
+ `table_fields` longtext COMMENT 'Data table fields, multiple are separated by half-width commas, required for increment',
+ `data_sql` longtext COMMENT 'SQL statement to collect source data, required for full amount',
+ `crontab` varchar(56) DEFAULT NULL COMMENT 'Timed scheduling expression, required for full amount',
+ `status` int(11) DEFAULT '0' COMMENT 'Data source status',
+ `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
);
@@ -443,22 +469,22 @@ CREATE TABLE `source_db_detail`
DROP TABLE IF EXISTS `source_file_basic`;
CREATE TABLE `source_file_basic`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'ID',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id',
- `is_hybrid_source` tinyint(1) DEFAULT '0' COMMENT 'Whether to mix data sources',
- `is_table_mapping` tinyint(1) DEFAULT '0' COMMENT 'Is there a table name mapping',
- `date_offset` int(4) DEFAULT '0' COMMENT 'Time offset\n',
- `date_offset_unit` varchar(2) DEFAULT 'H' COMMENT 'time offset unit',
- `file_rolling_type` varchar(2) DEFAULT 'H' COMMENT 'file rolling type',
- `upload_max_size` int(4) DEFAULT '120' COMMENT 'upload maximum size',
- `need_compress` tinyint(1) DEFAULT '0' COMMENT 'whether need compress',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'delete switch',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'temp view',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'ID',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Data stream id',
+ `is_hybrid_source` tinyint(1) DEFAULT '0' COMMENT 'Whether to mix data sources',
+ `is_table_mapping` tinyint(1) DEFAULT '0' COMMENT 'Is there a table name mapping',
+ `date_offset` int(4) DEFAULT '0' COMMENT 'Time offset\n',
+ `date_offset_unit` varchar(2) DEFAULT 'H' COMMENT 'Time offset unit',
+ `file_rolling_type` varchar(2) DEFAULT 'H' COMMENT 'File rolling type',
+ `upload_max_size` int(4) DEFAULT '120' COMMENT 'Upload maximum size',
+ `need_compress` tinyint(1) DEFAULT '0' COMMENT 'Whether need compress',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Delete switch',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'temp view',
PRIMARY KEY (`id`)
);
@@ -468,26 +494,26 @@ CREATE TABLE `source_file_basic`
DROP TABLE IF EXISTS `source_file_detail`;
CREATE TABLE `source_file_detail`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
`inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `access_type` varchar(20) DEFAULT 'Agent' COMMENT 'Collection type, there are Agent, DataProxy client, LoadProxy, the file can only be Agent temporarily',
- `server_name` varchar(64) DEFAULT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
- `ip` varchar(128) DEFAULT NULL COMMENT 'Data source IP address',
- `port` int(11) DEFAULT NULL COMMENT 'Data source port number',
- `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
- `issue_type` varchar(10) DEFAULT 'SSH' COMMENT 'Issuing method, there are SSH, TCS',
- `username` varchar(32) DEFAULT NULL COMMENT 'User name of the data source IP host',
- `password` varchar(64) DEFAULT NULL COMMENT 'The password corresponding to the above user name',
- `file_path` varchar(256) DEFAULT NULL COMMENT 'File path, supports regular matching',
- `status` int(11) DEFAULT '0' COMMENT 'Data source status',
- `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save unsubmitted and unapproved intermediate data after modification',
+ `access_type` varchar(20) DEFAULT 'Agent' COMMENT 'Collection type, there are Agent, DataProxy client, LoadProxy, the file can only be Agent temporarily',
+ `server_name` varchar(64) NOT NULL COMMENT 'The name of the data source service. If it is empty, add configuration through the following fields',
+ `ip` varchar(128) NOT NULL COMMENT 'Data source IP address',
+ `port` int(11) NOT NULL COMMENT 'Data source port number',
+ `is_inner_ip` tinyint(1) DEFAULT '0' COMMENT 'Whether it is intranet, 0: no, 1: yes',
+ `issue_type` varchar(10) DEFAULT 'SSH' COMMENT 'Issuing method, there are SSH, TCS',
+ `username` varchar(32) NOT NULL COMMENT 'User name of the data source IP host',
+ `password` varchar(64) NOT NULL COMMENT 'The password corresponding to the above user name',
+ `file_path` varchar(256) NOT NULL COMMENT 'File path, supports regular matching',
+ `status` int(11) DEFAULT '0' COMMENT 'Data source status',
+ `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
PRIMARY KEY (`id`)
);
@@ -514,33 +540,33 @@ DROP TABLE IF EXISTS `storage_hive`;
DROP TABLE IF EXISTS `storage_hive`;
CREATE TABLE `storage_hive`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
- `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
- `jdbc_url` varchar(255) DEFAULT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
- `username` varchar(128) DEFAULT NULL COMMENT 'Username',
- `password` varchar(255) DEFAULT NULL COMMENT 'User password',
- `db_name` varchar(128) DEFAULT NULL COMMENT 'Target database name',
- `table_name` varchar(128) DEFAULT NULL COMMENT 'Target data table name',
- `primary_partition` varchar(255) DEFAULT 'dt' COMMENT 'primary partition field',
- `secondary_partition` varchar(256) DEFAULT NULL COMMENT 'secondary partition field',
- `partition_type` varchar(10) DEFAULT NULL COMMENT 'The partition type, there are: H-by hour, D-by day, W-by week, M-by month, O-one-time, R-non-periodical',
- `file_format` varchar(15) DEFAULT 'TextFile' COMMENT 'The stored table format, TextFile, RCFile, SequenceFile, Avro',
- `encoding_type` varchar(255) DEFAULT NULL COMMENT 'data encoding',
- `field_splitter` varchar(10) DEFAULT NULL COMMENT 'field separator',
- `hdfs_default_fs` varchar(255) DEFAULT NULL COMMENT 'HDFS defaultFS, such as "hdfs://127.0.0.1:9000"',
- `warehouse_dir` varchar(250) DEFAULT '/user/hive/warehouse' COMMENT 'Hive table storage path on HDFS, such as "/user/hive/warehouse"',
- `usage_interval` varchar(10) DEFAULT NULL COMMENT 'The amount of time that Sort collected data will land on Hive, there are 10M, 15M, 30M, 1H, 1D',
- `storage_period` int(5) DEFAULT '10' COMMENT 'Data storage period, unit: day',
- `status` int(11) DEFAULT '0' COMMENT 'status',
- `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
- `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
- `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
- `opt_log` varchar(5000) DEFAULT NULL COMMENT 'Background operation log',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `inlong_group_id` varchar(128) NOT NULL COMMENT 'Owning business group id',
+ `inlong_stream_id` varchar(128) NOT NULL COMMENT 'Owning data stream id',
+ `jdbc_url` varchar(255) NOT NULL COMMENT 'Hive JDBC connection URL, such as "jdbc:hive2://127.0.0.1:10000"',
+ `username` varchar(128) NOT NULL COMMENT 'Username',
+ `password` varchar(255) NOT NULL COMMENT 'User password',
+ `db_name` varchar(128) NOT NULL COMMENT 'Target database name',
+ `table_name` varchar(128) NOT NULL COMMENT 'Target data table name',
+ `primary_partition` varchar(255) DEFAULT 'dt' COMMENT 'primary partition field',
+ `secondary_partition` varchar(256) DEFAULT NULL COMMENT 'secondary partition field',
+ `partition_type` varchar(10) DEFAULT NULL COMMENT 'The partition type, there are: H-by hour, D-by day, W-by week, M-by month, O-one-time, R-non-periodical',
+ `file_format` varchar(15) DEFAULT 'TextFile' COMMENT 'The stored table format, TextFile, RCFile, SequenceFile, Avro',
+ `encoding_type` varchar(255) DEFAULT NULL COMMENT 'data encoding',
+ `field_splitter` varchar(10) DEFAULT NULL COMMENT 'field separator',
+ `hdfs_default_fs` varchar(255) DEFAULT NULL COMMENT 'HDFS defaultFS, such as "hdfs://127.0.0.1:9000"',
+ `warehouse_dir` varchar(250) DEFAULT '/user/hive/warehouse' COMMENT 'Hive table storage path on HDFS, such as "/user/hive/warehouse"',
+ `usage_interval` varchar(10) DEFAULT NULL COMMENT 'The amount of time that Sort collected data will land on Hive, there are 10M, 15M, 30M, 1H, 1D',
+ `storage_period` int(5) DEFAULT '10' COMMENT 'Data storage period, unit: day',
+ `status` int(11) DEFAULT '0' COMMENT 'status',
+ `previous_status` int(11) DEFAULT '0' COMMENT 'Previous status',
+ `is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) DEFAULT NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
+ `temp_view` text DEFAULT NULL COMMENT 'Temporary view, used to save un-submitted and unapproved intermediate data after modification',
+ `opt_log` varchar(5000) DEFAULT NULL COMMENT 'Background operation log',
PRIMARY KEY (`id`)
);
@@ -550,13 +576,13 @@ CREATE TABLE `storage_hive`
DROP TABLE IF EXISTS `storage_hive_field`;
CREATE TABLE `storage_hive_field`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
- `source_field_name` varchar(20) DEFAULT NULL COMMENT 'source field name',
- `source_field_type` varchar(20) DEFAULT NULL COMMENT 'source field type',
- `field_name` varchar(20) DEFAULT NULL COMMENT 'field name',
- `field_type` varchar(20) DEFAULT NULL COMMENT 'field type',
- `field_comment` varchar(2000) DEFAULT NULL COMMENT 'Field description',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `storage_id` int(11) NOT NULL COMMENT 'Hive data storage id',
+ `source_field_name` varchar(20) NOT NULL COMMENT 'source field name',
+ `source_field_type` varchar(20) NOT NULL COMMENT 'source field type',
+ `field_name` varchar(20) NOT NULL COMMENT 'field name',
+ `field_type` varchar(20) NOT NULL COMMENT 'field type',
+ `field_comment` varchar(2000) DEFAULT '' COMMENT 'Field description',
`is_required` tinyint(1) DEFAULT NULL COMMENT 'Is it required, 0: not necessary, 1: required',
`bon_field_path` varchar(256) DEFAULT NULL COMMENT 'BON field path',
`bon_field_type` varchar(64) DEFAULT NULL COMMENT 'BON field type',
@@ -573,15 +599,15 @@ CREATE TABLE `storage_hive_field`
DROP TABLE IF EXISTS `task`;
CREATE TABLE `task`
(
- `id` bigint(20) NOT NULL,
- `taskflow_id` bigint(20) DEFAULT NULL COMMENT 'Owning task flow id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_id` bigint(20) NOT NULL COMMENT 'Owning task flow id',
`task_def_id` bigint(20) DEFAULT NULL COMMENT 'task definition id',
- `task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
+ `task_name` varchar(255) NOT NULL COMMENT 'task name',
`status` varchar(255) DEFAULT NULL COMMENT 'task status',
`post_param` varchar(255) DEFAULT NULL COMMENT 'Task parameters',
`resultmsg` varchar(1000) DEFAULT NULL COMMENT 'Execution result log',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(0) DEFAULT NULL COMMENT 'last modified person',
PRIMARY KEY (`id`)
@@ -593,13 +619,13 @@ CREATE TABLE `task`
DROP TABLE IF EXISTS `task_def`;
CREATE TABLE `task_def`
(
- `id` bigint(20) NOT NULL,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Task flow definition id',
+ `id` bigint(20) NOT NULL,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Task flow definition id',
`parent_id` bigint(20) DEFAULT NULL COMMENT 'parent task id',
`implclass` varchar(255) DEFAULT NULL COMMENT 'task processing flow class',
`task_name` varchar(255) DEFAULT NULL COMMENT 'task name',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` datetime DEFAULT NULL COMMENT 'last modified person',
`delivery_id` bigint(20) DEFAULT NULL COMMENT 'Task push method',
@@ -612,11 +638,11 @@ CREATE TABLE `task_def`
DROP TABLE IF EXISTS `taskflow`;
CREATE TABLE `taskflow`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `taskflow_def_id` bigint(20) DEFAULT NULL COMMENT 'Taskflow definition id',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `taskflow_def_id` bigint(20) NOT NULL COMMENT 'Taskflow definition id',
`status` varchar(255) DEFAULT NULL COMMENT 'status',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
+ `create_time` datetime DEFAULT NULL COMMENT 'Create time',
`update_time` datetime DEFAULT NULL COMMENT 'last modified time',
`update_by` varchar(255) DEFAULT NULL COMMENT 'last modified person',
`event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
@@ -629,11 +655,11 @@ CREATE TABLE `taskflow`
DROP TABLE IF EXISTS `taskflow_def`;
CREATE TABLE `taskflow_def`
(
- `id` bigint(20) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) DEFAULT NULL COMMENT 'Workflow definition name',
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) NOT NULL COMMENT 'Workflow definition name',
`descrip` varchar(255) DEFAULT NULL COMMENT 'Workflow function description',
- `create_time` datetime DEFAULT NULL COMMENT 'create time',
- `create_by` varchar(255) DEFAULT NULL COMMENT 'creator',
+ `create_time` datetime NOT NULL COMMENT 'Create time',
+ `create_by` varchar(255) NOT NULL COMMENT 'creator',
`isValid` int(11) DEFAULT NULL COMMENT 'logical deletion',
`trigger_event` varchar(255) DEFAULT NULL COMMENT 'trigger event',
PRIMARY KEY (`id`)
@@ -650,7 +676,7 @@ CREATE TABLE `user`
`password` varchar(64) NOT NULL COMMENT 'password md5',
`account_type` int(11) NOT NULL DEFAULT '1' COMMENT 'account type, 0-manager 1-normal',
`due_date` datetime DEFAULT NULL COMMENT 'due date for account',
- `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
`create_by` varchar(255) NOT NULL COMMENT 'create by sb.',
`update_by` varchar(255) DEFAULT NULL COMMENT 'update by sb.',
@@ -694,7 +720,7 @@ CREATE TABLE `wf_approver`
`approvers` varchar(1024) NOT NULL COMMENT 'Approvers, separated by commas',
`creator` varchar(64) NOT NULL COMMENT 'creator',
`modifier` varchar(64) NOT NULL COMMENT 'modifier',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
`is_deleted` int(11) DEFAULT '0' COMMENT 'Whether to delete, 0 is not deleted, if greater than 0, delete',
PRIMARY KEY (`id`)
@@ -718,8 +744,8 @@ CREATE TABLE `wf_event_log`
`process_inst_id` int(11) NOT NULL,
`process_name` varchar(255) DEFAULT NULL COMMENT 'Process name',
`process_display_name` varchar(255) NOT NULL COMMENT 'Process name',
- `group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id',
- `task_inst_id` int(11) DEFAULT NULL COMMENT 'Task id',
+ `inlong_group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id',
+ `task_inst_id` int(11) DEFAULT NULL COMMENT 'Task ID',
`element_name` varchar(255) NOT NULL COMMENT 'The name of the component that triggered the event',
`element_display_name` varchar(255) NOT NULL COMMENT 'Chinese name of the component that triggered the event',
`event_type` varchar(64) NOT NULL COMMENT 'Event type: process event/task event',
@@ -741,19 +767,19 @@ CREATE TABLE `wf_event_log`
DROP TABLE IF EXISTS `wf_process_instance`;
CREATE TABLE `wf_process_instance`
(
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL COMMENT 'process name',
- `display_name` varchar(255) NOT NULL COMMENT 'Process display name',
- `type` varchar(255) DEFAULT NULL COMMENT 'Process classification',
- `title` varchar(255) DEFAULT NULL COMMENT 'Process title',
- `group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id: to facilitate related business',
- `applicant` varchar(255) NOT NULL COMMENT 'applicant',
- `state` varchar(64) NOT NULL COMMENT 'state',
- `form_data` mediumtext COMMENT 'form information',
- `start_time` datetime NOT NULL COMMENT 'start time',
- `end_time` datetime DEFAULT NULL COMMENT 'End event',
- `ext` text COMMENT 'Extended information-json',
- `hidden` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Is it hidden',
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) NOT NULL COMMENT 'process name',
+ `display_name` varchar(255) NOT NULL COMMENT 'Process display name',
+ `type` varchar(255) DEFAULT NULL COMMENT 'Process classification',
+ `title` varchar(255) DEFAULT NULL COMMENT 'Process title',
+ `inlong_group_id` varchar(128) DEFAULT NULL COMMENT 'Business group id: to facilitate related business',
+ `applicant` varchar(255) NOT NULL COMMENT 'applicant',
+ `state` varchar(64) NOT NULL COMMENT 'state',
+ `form_data` mediumtext COMMENT 'form information',
+ `start_time` datetime NOT NULL COMMENT 'start time',
+ `end_time` datetime DEFAULT NULL COMMENT 'End event',
+ `ext` text COMMENT 'Extended information-text',
+ `hidden` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Is it hidden',
PRIMARY KEY (`id`)
);
@@ -778,7 +804,7 @@ CREATE TABLE `wf_task_instance`
`form_data` mediumtext COMMENT 'form information submitted by the current task',
`start_time` datetime NOT NULL COMMENT 'start time',
`end_time` datetime DEFAULT NULL COMMENT 'End time',
- `ext` text COMMENT 'Extended information-json',
+ `ext` text COMMENT 'Extended information-text',
PRIMARY KEY (`id`)
);
@@ -790,17 +816,17 @@ CREATE TABLE `cluster_set`
(
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
- `cn_name` varchar(256) DEFAULT NULL COMMENT 'Chinese display name',
- `description` varchar(256) DEFAULT NULL COMMENT 'ClusterSet Introduction',
- `middleware_type` varchar(10) DEFAULT 'Pulsar' COMMENT 'The middleware type of data storage, high throughput: Pulsar',
- `in_charges` varchar(512) DEFAULT NULL COMMENT 'Name of responsible person, separated by commas',
- `followers` varchar(512) DEFAULT NULL COMMENT 'List of names of business followers, separated by commas',
+ `cn_name` varchar(256) COMMENT 'Chinese display name',
+ `description` varchar(256) COMMENT 'ClusterSet Introduction',
+ `middleware_type` varchar(10) DEFAULT 'TUBE' COMMENT 'The middleware type of message queue, high throughput: TUBE, high consistency: PULSAR',
+ `in_charges` varchar(512) COMMENT 'Name of responsible person, separated by commas',
+ `followers` varchar(512) COMMENT 'List of names of business followers, separated by commas',
`status` int(11) DEFAULT '21' COMMENT 'ClusterSet status',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
- `creator` varchar(64) DEFAULT NULL COMMENT 'creator name',
- `modifier` varchar(64) DEFAULT NULL COMMENT 'modifier name',
- `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
- `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
+ `creator` varchar(64) NOT NULL COMMENT 'Creator name',
+ `modifier` varchar(64) NULL COMMENT 'Modifier name',
+ `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time',
+ `modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Modify time',
PRIMARY KEY (`id`)
);
@@ -810,8 +836,8 @@ CREATE TABLE `cluster_set`
DROP TABLE IF EXISTS `cluster_set_inlongid`;
CREATE TABLE `cluster_set_inlongid`
(
- `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
- `set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
+ `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
+ `set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`inlong_group_id` varchar(128) NOT NULL COMMENT 'Business group id, filled in by the user, undeleted ones cannot be repeated',
PRIMARY KEY (`id`)
);
@@ -838,7 +864,7 @@ CREATE TABLE `cache_cluster_ext`
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Incremental primary key',
`cluster_name` varchar(128) NOT NULL COMMENT 'CacheCluster name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
@@ -907,7 +933,7 @@ CREATE TABLE `flume_source_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSource name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
@@ -936,7 +962,7 @@ CREATE TABLE `flume_channel_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeChannel name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
@@ -966,10 +992,10 @@ CREATE TABLE `flume_sink_ext`
`parent_name` varchar(128) NOT NULL COMMENT 'FlumeSink name, English, numbers and underscore',
`set_name` varchar(128) NOT NULL COMMENT 'ClusterSet name, English, numbers and underscore',
`key_name` varchar(64) NOT NULL COMMENT 'Configuration item name',
- `key_value` varchar(256) DEFAULT NULL COMMENT 'The value of the configuration item',
+ `key_value` varchar(256) NULL COMMENT 'The value of the configuration item',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT 'Whether to delete, 0: not deleted, 1: deleted',
`modify_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'modify time',
PRIMARY KEY (`id`)
);
-SET FOREIGN_KEY_CHECKS = 1;
\ No newline at end of file
+SET FOREIGN_KEY_CHECKS = 1;