You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/09/19 03:18:09 UTC
[inlong-website] branch master updated: [INLONG-560][Doc] Update sort-standalone docs (#561)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong-website.git
The following commit(s) were added to refs/heads/master by this push:
new 823796d789 [INLONG-560][Doc] Update sort-standalone docs (#561)
823796d789 is described below
commit 823796d789d4e8a7183de62e9fc905e8fc938320
Author: vernedeng <de...@pku.edu.cn>
AuthorDate: Mon Sep 19 11:18:04 2022 +0800
[INLONG-560][Doc] Update sort-standalone docs (#561)
---
.../sort-standalone/elasticsearch_example.md | 145 ++++--------
docs/modules/sort-standalone/hive_example.md | 262 +++++++++++++--------
docs/modules/sort-standalone/overview.md | 15 +-
docs/modules/sort-standalone/quick_start.md | 249 +++++++-------------
4 files changed, 301 insertions(+), 370 deletions(-)
diff --git a/docs/modules/sort-standalone/elasticsearch_example.md b/docs/modules/sort-standalone/elasticsearch_example.md
index d48b78a4f5..2c47f4caf4 100644
--- a/docs/modules/sort-standalone/elasticsearch_example.md
+++ b/docs/modules/sort-standalone/elasticsearch_example.md
@@ -27,8 +27,8 @@ sortSource.type=org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource
sortClusterConfig.type=file
sortClusterConfig.file=SortClusterConfig.conf
sortSourceConfig.QueryConsumeConfigType=file
-#sortTaskId.conf
+# manager config example
#sortClusterConfig.type=manager
#sortSourceConfig.QueryConsumeConfigType=manager
#managerUrlLoaderType=org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader
@@ -47,16 +47,15 @@ sortSourceConfig.QueryConsumeConfigType=file
"idParams": [{
"indexNamePattern": "inlong0fc00000046_{yyyyMMdd}",
"contentOffset": "0",
- "inlongGroupId": "atta",
- "fieldOffset": "2",
+ "inlongGroupId": "testgroup",
+ "fieldOffset": "0",
"fieldNames": "ftime extinfo t1 t2 t3 t4",
"inlongStreamId": "0fc00000046",
"separator": "|"
}],
"sinkParams": {
- "httpHosts": "11.187.135.221:9200",
- "password": "yingyan@ES",
- "auditSetName": "es-rmrv7g7a",
+ "httpHosts": "ip:port",
+ "password": "password",
"bulkSizeMb": "10",
"flushInterval": "60",
"keywordMaxLength": "32767",
@@ -78,11 +77,11 @@ sortSourceConfig.QueryConsumeConfigType=file
"sortTaskId": "sid_es_v3",
"cacheZones": {
"pc_atta6th_sz1": {
- "zoneName": "pc_atta6th_sz1",
- "serviceUrl": "http://9.139.53.86:8080",
- "authentication": "eyJrZXlJZCI6InB1bHNhci04MnhhN24zZWs1ZHciLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwdWxzYXItODJ4YTduM2VrNWR3X2FkbWluIn0.D5H_j8UQk8KYWHw_mzq2HmR393SnbL5Gz7JYCANBPnI",
+ "zoneName": "${PULSAR_CLUSTER_NAME}",
+ "serviceUrl": "http://${PULSAR_IP}:${PULSAR_PORT}",
+ "authentication": "${PULSAR_AUTH}",
"topics": [{
- "topic": "pulsar-82xa7n3ek5dw/atta/atta_topic_1",
+ "topic": "${TENANT/NAMESPACE/TOPIC}",
"partitionCnt": 10,
"topicProperties": {}
}],
@@ -93,103 +92,59 @@ sortSourceConfig.QueryConsumeConfigType=file
}
```
-## Modify configuration file:conf/common.properties
-
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterId | Y | NA | inlong-sort-standalone cluster id |
-|nodeId | N | Local IP | Current node id |
-|metricDomains | N | Sort | domain name of metric |
-|metricDomains.Sort.domainListeners | N | org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener | class name list of metric listener, separated by space |
-|metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric data(millisecond) |
-|prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
-|sortChannel.type | N | org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class name |
-|sortSink.type | Y | NA | Sink class name |
-|sortSource.type | N | org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource | Source class name |
-|sortClusterConfig.type | N | manager | Loader source of cluster configuration data: [file,manager,UserDefinedClassName]. |
-|sortClusterConfig.file | N | SortClusterConfig.conf | File name in class resource when sortClusterConfig.type=file. |
-|sortClusterConfig.managerUrl | N | NA | The parameter is the cluster configuration URL of InlongManager when sortClusterConfig.type=manager. <br/>For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
-|sortSourceConfig.QueryConsumeConfigType | N | manager | Loader source of sort task configuration data: [file,manager,UserDefinedClassName]. <br/>Sort task configuration file is ${sortTaskId}.conf in the class resource when sortSourceConfig.QueryConsumeConfigType=file. |
-|sortSourceConfig.managerUrl | N | NA | The parameter is the sort task configuration URL of InlongManager when sortClusterConfig.type=manager. <br/>For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getSortSource |
-
-## Modify configuration file: SortClusterConfig.conf
-
-- Get cluster configuration data from the file:SortClusterConfig.conf in classpath, it can not support online updating.
-- Get cluster configuration data from InlongManager URL, it can support online updating.
-
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|clusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTasks | Y | JsonArray<SortTaskConfig> |NA | Sort task list |
-
-### Modify configuration: SortTaskConfig
-
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|name | Y | NA | sort task name |
-|type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"), ELASTICSEARCH("elasticsearch"), UNKNOWN("n") |
-|idParams | Y | NA | Inlong DataStream configuration |
-|sinkParams | Y | NA | Sort task parameters |
-
### Modify configuration: idParams of Elasticsearch sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|separator | Y | NA | separator of Inlong datastream in data source |
-|fieldNames | Y | NA | field name list of Elasticsearch index, separated by space. |
-|indexNamePattern | Y | NA | index name pattern of Elasticsearch,date time variable include {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm}. |
-|contentOffset | Y | NA | field index offset of source content |
-|fieldOffset | Y | NA | offset of Elasticsearch index field name list |
+| Parameter | Required | DefaultValue | Remark |
+|------------------|-----------|----------------|--------------------------------------------------------------------------------------------------------|
+| inlongGroupId | Y | NA | inlongGroupId |
+| inlongStreamId | Y | NA | inlongStreamId |
+| separator | Y | NA | separator of Inlong datastream in data source |
+| fieldNames | Y | NA | field name list of Elasticsearch index, separated by space. |
+| indexNamePattern | Y | NA | index name pattern of Elasticsearch,date time variable include {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm}. |
+| contentOffset | Y | NA | field index offset of source content |
+| fieldOffset | Y | NA | offset of Elasticsearch index field name list |
### Modify configuration: sinkParams of Elasticsearch sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|httpHosts | Y | NA | Hosts of Elasticsearch |
-|username | Y | NA | Username of Elasticsearch |
-|password | Y | NA | Password of Elasticsearch |
-|isUseIndexId | N | false | Create index id or not |
-|bulkSizeMb | N | 10 | Max content size per bulk(MB) |
-|flushInterval | N | 60 | Max interval between flushing operation(Second) |
-|keywordMaxLength | N | 32767 | Max keyword length(Byte) |
-|bulkAction | N | 4000 | Max index request per bulk |
-|maxConnect | N | 10 | Max opening HTTP connect |
-|concurrentRequests | N | 5 | Max concurrent requests per HTTP connect |
-
-## Modify configuration file: sid_es_v3.conf
-
-- The file name include sort task name plus the postfix ".conf".
-- Get the configuration data from the file in classpath, it can not support online updating.
-- Get the configuration data from InlongManager URL, it can support online updating.
+| Parameter | Required | DefaultValue | Remark |
+|--------------------|-------------|----------------|-------------------------------------------------|
+| httpHosts | Y | NA | Hosts of Elasticsearch |
+| username | Y | NA | Username of Elasticsearch |
+| password | Y | NA | Password of Elasticsearch |
+| isUseIndexId | N | false | Create index id or not |
+| bulkSizeMb | N | 10 | Max content size per bulk(MB) |
+| flushInterval | N | 60 | Max interval between flushing operation(Second) |
+| keywordMaxLength | N | 32767 | Max keyword length(Byte) |
+| bulkAction | N | 4000 | Max index request per bulk |
+| maxConnect | N | 10 | Max opening HTTP connect |
+| concurrentRequests | N | 5 | Max concurrent requests per HTTP connect |
+
### Modify configuration: sid_es_v3.conf
+- The file name include sort task name plus the postfix ".conf".
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|sortClusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTaskId | Y | String |NA | Sort task name |
-|cacheZones | Y | JsonObject<String, JsonObject> |NA | Cache cluster list, Map<cacheClusterName, CacheCluster> |
+| Parameter | Required | Type | DefaultValue | Remark |
+|-----------------|------------|--------------------------------------|---------------|---------------------------------------------------------------|
+| sortClusterName | Y | String | NA | inlong-sort-standalone cluster id |
+| sortTaskId | Y | String | NA | Sort task name |
+| cacheZones | Y | JsonObject<String, JsonObject> | NA | Cache cluster list, Map<cacheClusterName, CacheCluster> |
### Modify configuration: CacheCluster
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|zoneName | Y |String | NA | cache cluster name |
-|zoneType | Y | String |NA | [pulsar,tube,kafka] |
-|serviceUrl | Y | String |NA | Pulsar serviceUrl or Kafka broker list |
-|authentication | Y | String |NA | Pulsar authentication |
-|cacheZoneProperties | N | Map<String,String> |NA | Cache consumer configuration |
-|topics | N | List<Topic> |NA | Topic list of Cache consumer |
+| Parameter | Required | Type | DefaultValue | Remark |
+|---------------------|-----------|--------------------------|-----------------|----------------------------------------|
+| zoneName | Y | String | NA | cache cluster name |
+| zoneType | Y | String | NA | [pulsar,tube,kafka] |
+| serviceUrl | Y | String | NA | Pulsar serviceUrl or Kafka broker list |
+| authentication | Y | String | NA | Pulsar authentication |
+| cacheZoneProperties | N | Map<String,String> | NA | Cache consumer configuration |
+| topics | N | List<Topic> | NA | Topic list of Cache consumer |
### Modify configuration: Topic
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|topic | Y |String | NA | cache topic name |
-|partitionCnt | Y | Integer |NA | cache topic partition count |
-|topicProperties | N | Map<String,String> |NA | Cache topic configuration |
+| Parameter | Required | Type | DefaultValue | Remark |
+|-----------------|-------------|--------------------------|----------------|-----------------------------|
+| topic | Y | String | NA | cache topic name |
+| partitionCnt | Y | Integer | NA | cache topic partition count |
+| topicProperties | N | Map<String,String> | NA | Cache topic configuration |
-## Start inlong-sort-standalone application
-At last, execute the shell file "./bin/sort-start.sh" for starting sort-standalone, you can check the log file "sort.log".
diff --git a/docs/modules/sort-standalone/hive_example.md b/docs/modules/sort-standalone/hive_example.md
index 218f8face0..be6bc67be2 100644
--- a/docs/modules/sort-standalone/hive_example.md
+++ b/docs/modules/sort-standalone/hive_example.md
@@ -27,8 +27,8 @@ sortSource.type=org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource
sortClusterConfig.type=file
sortClusterConfig.file=SortClusterConfig.conf
sortSourceConfig.QueryConsumeConfigType=file
-#sortTaskId.conf
+# manager config example
#sortClusterConfig.type=manager
#sortSourceConfig.QueryConsumeConfigType=manager
#managerUrlLoaderType=org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader
@@ -38,40 +38,101 @@ sortSourceConfig.QueryConsumeConfigType=file
```
### Example: conf/SortClusterConfig.conf
-
```
{
- "clusterName": "hivev3-sz-sz1",
- "sortTasks": [{
- "name": "sid_hive_inlong6th_v3",
- "type": "HIVE",
- "idParams": [{
- "inlongGroupId": "atta",
- "inlongStreamId": "0fc00000046",
- "separator": "|",
- "partitionIntervalMs": 3600000,
- "idRootPath": "/user/hive/warehouse/t_inlong_v1_0fc00000046",
- "partitionSubPath": "/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName": "t_inlong_v1_0fc00000046",
- "partitionFieldName": "dt",
- "partitionFieldPattern": "yyyyMMddHH",
- "msgTimeFieldPattern": "yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour": 8
- }],
- "sinkParams": {
- "hdfsPath": "hdfs://10.160.139.123:9000",
- "maxFileOpenDelayMinute": "5",
- "tokenOvertimeMinute": "60",
- "maxOutputFileSizeGb": "2",
- "hiveJdbcUrl": "jdbc:hive2://10.160.142.179:10000",
- "hiveDatabase": "default",
- "hiveUsername": "hive",
- "hivePassword": "hive"
- }
- }]
+ "data":{
+ "clusterName":"hivev3-sz-sz1",
+ "sortTasks":[
+ {
+ "idParams":[
+ {
+ "inlongGroupId":"0fc00000046",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+ "idRootPath":"/user/hive/warehouse/t_inlong_v1_0fc00000046",
+ "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+ "hiveTableName":"t_inlong_v1_0fc00000046",
+ "partitionFieldName":"dt",
+ "partitionFieldPattern":"yyyyMMddHH",
+ "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"03600000045",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+ "idRootPath":"/user/hive/warehouse/t_inlong_v1_03600000045",
+ "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+ "hiveTableName":"t_inlong_v1_03600000045",
+ "partitionFieldName":"dt",
+ "partitionFieldPattern":"yyyyMMddHH",
+ "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"05100054990",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+ "idRootPath":"/user/hive/warehouse/t_inlong_v1_05100054990",
+ "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+ "hiveTableName":"t_inlong_v1_05100054990",
+ "partitionFieldName":"dt",
+ "partitionFieldPattern":"yyyyMMddHH",
+ "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"09c00014434",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+ "idRootPath":"/user/hive/warehouse/t_inlong_v1_09c00014434",
+ "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+ "hiveTableName":"t_inlong_v1_09c00014434",
+ "partitionFieldName":"dt",
+ "partitionFieldPattern":"yyyyMMddHH",
+ "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"0c900035509",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+ "idRootPath":"/user/hive/warehouse/t_inlong_v1_0c900035509",
+ "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+ "hiveTableName":"t_inlong_v1_0c900035509",
+ "partitionFieldName":"dt",
+ "partitionFieldPattern":"yyyyMMddHH",
+ "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ }
+ ],
+ "name":"sid_hive_inlong6th_v3",
+ "sinkParams":{
+ "hdfsPath":"hdfs://127.0.0.1:9000",
+ "maxFileOpenDelayMinute":"5",
+ "tokenOvertimeMinute":"60",
+ "maxOutputFileSizeGb":"2",
+ "hiveJdbcUrl":"jdbc:hive2://127.0.0.2:10000",
+ "hiveDatabase":"default",
+ "hiveUsername":"hive",
+ "hivePassword":"hive"
+ },
+ "type":"HIVE"
+ }
+ ]
+ },
+ "errCode":0,
+ "md5":"md5",
+ "result":true
}
```
+
### Example: conf/sid_hive_inlong6th_v3.conf
```
@@ -97,103 +158,100 @@ sortSourceConfig.QueryConsumeConfigType=file
## Modify configuration file:conf/common.properties
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterId | Y | NA | inlong-sort-standalone cluster id |
-|nodeId | N | Local IP | Current node id |
-|metricDomains | N | Sort | domain name of metric |
-|metricDomains.Sort.domainListeners | N | org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener | class name list of metric listener, separated by space |
-|metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric data(millisecond) |
-|prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
-|sortChannel.type | N | org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class name |
-|sortSink.type | Y | NA | Sink class name |
-|sortSource.type | N | org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource | Source class name |
-|sortClusterConfig.type | N | manager | Loader source of cluster configuration data: [file,manager,UserDefinedClassName]. |
-|sortClusterConfig.file | N | SortClusterConfig.conf | File name in class resource when sortClusterConfig.type=file. |
-|sortClusterConfig.managerUrl | N | NA | The parameter is the cluster configuration URL of InlongManager when sortClusterConfig.type=manager. <br/>For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
-|sortSourceConfig.QueryConsumeConfigType | N | manager | Loader source of sort task configuration data: [file,manager,UserDefinedClassName]. <br/>Sort task configuration file is ${sortTaskId}.conf in the class resource when sortSourceConfig.QueryConsumeConfigType=file. |
-|sortSourceConfig.managerUrl | N | NA | The parameter is the sort task configuration URL of InlongManager when sortClusterConfig.type=manager. <br/>For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getSortSource |
+| Parameter | Required | DefaultValue | Remark |
+|-----------------------------------------|------------|-------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| clusterId | Y | NA | inlong-sort-standalone cluster id |
+| nodeId | N | Local IP | Current node id |
+| metricDomains | N | Sort | domain name of metric |
+| metricDomains.Sort.domainListeners | N | org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener | class name list of metric listener, separated by space |
+| metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric data(millisecond) |
+| prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
+| sortChannel.type | N | org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class name |
+| sortSink.type | Y | NA | Sink class name |
+| sortSource.type | N | org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource | Source class name |
+| sortClusterConfig.type | N | manager | Loader source of cluster configuration data: [file,manager,UserDefinedClassName]. |
+| sortClusterConfig.file | N | SortClusterConfig.conf | File name in class resource when sortClusterConfig.type=file. |
+| sortClusterConfig.managerUrl | N | NA | The parameter is the cluster configuration URL of InlongManager when sortClusterConfig.type=manager. <br/>For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
+| sortSourceConfig.QueryConsumeConfigType | N | manager | Loader source of sort task configuration data: [file,manager,UserDefinedClassName]. <br/>Sort task configuration file is ${sortTaskId}.conf in the class resource when sortSourceConfig.QueryConsumeConfigType=file. |
+| sortSourceConfig.managerUrl | N | NA | The parameter is the sort task configuration URL of InlongManager when sortClusterConfig.type=manager. <br/>For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getSortSource |
## Modify configuration file: SortClusterConfig.conf
- Get cluster configuration data from the file:SortClusterConfig.conf in classpath, it can not support online updating.
- Get cluster configuration data from InlongManager URL, it can support online updating.
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|clusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTasks | Y | JsonArray<SortTaskConfig> |NA | Sort task list |
+| Parameter | Required | Type | DefaultValue | Remark |
+|-------------|------------|---------------------------------|----------------|-----------------------------------|
+| clusterName | Y | String | NA | inlong-sort-standalone cluster id |
+| sortTasks | Y | JsonArray<SortTaskConfig> | NA | Sort task list |
### Modify configuration: SortTaskConfig
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|name | Y | NA | sort task name |
-|type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n") |
-|idParams | Y | NA | Inlong DataStream configuration |
-|sinkParams | Y | NA | Sort task parameters |
+| Parameter | Required | DefaultValue | Remark |
+|-------------|-------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| name | Y | NA | sort task name |
+| type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n") |
+| idParams | Y | NA | Inlong DataStream configuration |
+| sinkParams | Y | NA | Sort task parameters |
### Modify configuration: idParams of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|separator | Y | NA | separator of Inlong datastream in data source |
-|partitionIntervalMs | N | 3600000 | partition interval(millisecond) |
-|idRootPath | Y | NA | HDFS root path of Inlong DataStream |
-|partitionSubPath | Y | NA | partition sub path of Inlong DataStream |
-|hiveTableName | Y | NA | Hive table name of Inlong DataStream |
-|partitionFieldName | N | dt | partition field name of Inlong DataStream |
-|partitionFieldPattern | Y | NA | Date format of partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
-|msgTimeFieldPattern | Y | NA | Date format of message generation time, it support Java date format |
-|maxPartitionOpenDelayHour | N | 8 | Max delay hour of partition(hour) |
+| Parameter | Required | DefaultValue | Remark |
+|---------------------------|------------|-----------------|--------------------------------------------------------------------------------------------|
+| inlongGroupId | Y | NA | inlongGroupId |
+| inlongStreamId | Y | NA | inlongStreamId |
+| separator | Y | NA | separator of Inlong datastream in data source |
+| partitionIntervalMs | N | 3600000 | partition interval(millisecond) |
+| idRootPath | Y | NA | HDFS root path of Inlong DataStream |
+| partitionSubPath | Y | NA | partition sub path of Inlong DataStream |
+| hiveTableName | Y | NA | Hive table name of Inlong DataStream |
+| partitionFieldName | N | dt | partition field name of Inlong DataStream |
+| partitionFieldPattern | Y | NA | Date format of partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
+| msgTimeFieldPattern | Y | NA | Date format of message generation time, it support Java date format |
+| maxPartitionOpenDelayHour | N | 8 | Max delay hour of partition(hour) |
### Modify configuration: sinkParams of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|hdfsPath | Y | NA | NameNode URL of HDFS |
-|maxFileOpenDelayMinute | N | 5 | Max writing delay minutes of simple HDFS file(minute) |
-|tokenOvertimeMinute | N | 60 | token overtime of Inlong Data Stream(minute) |
-|maxOutputFileSizeGb | N | 2 | Max file size of simple HDFS file(GB) |
-|hiveJdbcUrl | Y | NA | JDBC URL of Hive |
-|hiveDatabase | Y | NA | Hive database |
-|hiveUsername | Y | NA | Hive username |
-|hivePassword | Y | NA | Hive password |
+| Parameter | Required | DefaultValue | Remark |
+|------------------------|------------|-----------------|-------------------------------------------------------|
+| hdfsPath | Y | NA | NameNode URL of HDFS |
+| maxFileOpenDelayMinute | N | 5 | Max writing delay minutes of simple HDFS file(minute) |
+| tokenOvertimeMinute | N | 60 | token overtime of Inlong Data Stream(minute) |
+| maxOutputFileSizeGb | N | 2 | Max file size of simple HDFS file(GB) |
+| hiveJdbcUrl | Y | NA | JDBC URL of Hive |
+| hiveDatabase | Y | NA | Hive database |
+| hiveUsername | Y | NA | Hive username |
+| hivePassword | Y | NA | Hive password |
## Modify configuration file: sid_hive_inlong6th_v3.conf
-- The file name include sort task name plus the postfix ".conf".
-- Get the configuration data from the file in classpath, it can not support online updating.
-- Get the configuration data from InlongManager URL, it can support online updating.
+- The file name include sort task name plus the postfix ".conf".
### Modify configuration: sid_hive_inlong6th_v3.conf
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|sortClusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTaskId | Y | String |NA | Sort task name |
-|cacheZones | Y | JsonObject<String, JsonObject> |NA | Cache cluster list, Map<cacheClusterName, CacheCluster> |
+| Parameter | Required | Type | DefaultValue | Remark |
+|-----------------|-------------|--------------------------------------|----------------|---------------------------------------------------------------|
+| sortClusterName | Y | String | NA | inlong-sort-standalone cluster id |
+| sortTaskId | Y | String | NA | Sort task name |
+| cacheZones | Y | JsonObject<String, JsonObject> | NA | Cache cluster list, Map<cacheClusterName, CacheCluster> |
### Modify configuration: CacheCluster
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|zoneName | Y |String | NA | cache cluster name |
-|zoneType | Y | String |NA | [pulsar,tube,kafka] |
-|serviceUrl | Y | String |NA | Pulsar serviceUrl or Kafka broker list |
-|authentication | Y | String |NA | Pulsar authentication |
-|cacheZoneProperties | N | Map<String,String> |NA | Cache consumer configuration |
-|topics | N | List<Topic> |NA | Topic list of Cache consumer |
+| Parameter | Required | Type | DefaultValue | Remark |
+|---------------------|-------------|--------------------------|----------------|----------------------------------------|
+| zoneName | Y | String | NA | cache cluster name |
+| zoneType | Y | String | NA | [pulsar,tube,kafka] |
+| serviceUrl | Y | String | NA | Pulsar serviceUrl or Kafka broker list |
+| authentication | Y | String | NA | Pulsar authentication |
+| cacheZoneProperties | N | Map<String,String> | NA | Cache consumer configuration |
+| topics | N | List<Topic> | NA | Topic list of Cache consumer |
### Modify configuration: Topic
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|topic | Y |String | NA | cache topic name |
-|partitionCnt | Y | Integer |NA | cache topic partition count |
-|topicProperties | N | Map<String,String> |NA | Cache topic configuration |
+| Parameter | Required | Type | DefaultValue | Remark |
+|-----------------|------------|--------------------------|-----------------|-----------------------------|
+| topic | Y | String | NA | cache topic name |
+| partitionCnt | Y | Integer | NA | cache topic partition count |
+| topicProperties | N | Map<String,String> | NA | Cache topic configuration |
+
-## Start inlong-sort-standalone application
-At last, execute the shell file "./bin/sort-start.sh" for starting sort-standalone, you can check the log file "sort.log".
diff --git a/docs/modules/sort-standalone/overview.md b/docs/modules/sort-standalone/overview.md
index ebbb9f5964..b3e0b0632c 100644
--- a/docs/modules/sort-standalone/overview.md
+++ b/docs/modules/sort-standalone/overview.md
@@ -3,7 +3,7 @@ title: Overview
sidebar_position: 1
---
-## Overview
+## Introduction
Inlong sort standalone is a module responsible for consuming the data stream reported by users from the cache layer and distributing it to different data stores. It supports hive, elasticsearch, CLS and other data stores.
Inlong sort standalone relies on inlong manager to manage system metadata. Inlong sort standalone is deployed by cluster and aggregates and distributes tasks by target storage.
@@ -23,20 +23,21 @@ After the inlong data stream is distributed offline, it goes online again. If th
If the consumption location at the time of offline is no longer within the life cycle of the cache layer, consumption starts from the latest location of the cache layer.
### message queues supported by the cache layer
-- Inlong-tubemq
+- Inlong-Tube
- Apache Pulsar
+- Apache Kafka
### supported data storage
-- Apache hive (currently only supports sequence file format)
+- Apache Hive (currently only supports sequence file format)
- Apache Pulsar
- Apache Kafka
+- Elasticsearch
+- ClickHouse
+- Tencent CLS
### Future planning
-#### support more types of cache layer message queues
-Apache Kafka, etc.
-
#### support more kinds of data storage
-HBase, elasticsearch, etc.
+HBase, etc.
#### support more file formats written to Apache hive
Orc file, etc.
diff --git a/docs/modules/sort-standalone/quick_start.md b/docs/modules/sort-standalone/quick_start.md
index f72dbb9c71..482c63bf72 100644
--- a/docs/modules/sort-standalone/quick_start.md
+++ b/docs/modules/sort-standalone/quick_start.md
@@ -6,185 +6,102 @@ sidebar_position: 2
Module archive is in the directory:inlong-sort-standalone/sort-standalone-dist/target/, the archive file is apache-inlong-sort-standalone-${project.version}-bin.tar.gz.
## Start inlong-sort-standalone application
-At first, decompress the archive file, execute the shell file "./bin/sort-start.sh".
+- Decompress the archive file;
+- Prepare conf/common.properties (and other config files if choosing local file config mode);
+- Start SortStandalone ```./bin/sort-start.sh```
+- You can check the log file "sort.log".
## Configuration file:conf/common.properties
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterId | Y | NA | inlong-sort-standalone cluster id |
-|sortSource.type | N | org.apache.inlong.sort.standalone.source.readapi.ReadApiSource | Source class name |
-|sortChannel.type | N | org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class name |
-|sortSink.type | N | org.apache.inlong.sort.standalone.sink.hive.HiveSink | Sink class name |
-|sortClusterConfig.type | N | org.apache.inlong.sort.standalone.config.loader.ClassResourceSortClusterConfigLoader | Configuration data loader class name |
-|sortClusterConfig.managerPath | N | NA | For loader:org.apache.inlong.sort.standalone.config.loader.ManagerSortClusterConfigLoader, the parameter is the URL of InlongManager. For example:http://${manager ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
-|eventFormatHandler | N | org.apache.inlong.sort.standalone.sink.hive.DefaultEventFormatHandler | Formater class name |
-|maxThreads | N | 10 | sink thread number |
-|reloadInterval | N | 60000 | interval updating Configuration data(millisecond) |
-|processInterval | N | 100 | interval processing data(millisecond) |
-|metricDomains | N | Sort | domain name of metric |
-|metricDomains.Sort.domainListeners | N | org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener | class name list of metric listener, separated by space |
-|prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
-|metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric data(millisecond) |
+| Parameter | Required | DefaultValue | Remark |
+|-----------------------------------------|----------|----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|
+| clusterId | Y | NA | inlong-sort-standalone cluster id |
+| sortSource.type | N | org.apache.inlong.sort.standalone.source.readapi.SortSdkSource | Source class name |
+| sortChannel.type | N | org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class name |
+| sortSink.type | N | org.apache.inlong.sort.standalone.sink.hive.HiveSink | Sink class name |
+| sortClusterConfig.type | Y | manager | ClusterConfig configuration type. File config mode please choose: ***file*** |
+| sortSourceConfig.QueryConsumeConfigType | Y | manager | SourceConfig configuration type. File config mode please choose: ***file*** |
+| sortClusterConfig.file | N | SortClusterConfig.conf | Sort cluster config file name. ***Required in file config type.*** |
+| managerUrlLoaderType | N | org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader | The type of manager url loader. ***Required in manager config type.*** |
+| sortClusterConfig.managerUrl | N | http://${manager_ip}:{manager_port}/inlong/manager/openapi/sort/getClusterConfig | The manager config interface to cluster config. ***Required in manager config type.*** |
+ | sortSourceConfig.managerUrl | N | http://${manager_ip}:{manager_port}/inlong/manager/openapi/sort/getSortSource | The manager config interface to sort source config. ***Required in manager config mode.*** |
+| eventFormatHandler | N | org.apache.inlong.sort.standalone.sink.hive.DefaultEventFormatHandler | Formatter class name |
+| maxThreads | N | 10 | sink thread number |
+| reloadInterval | N | 60000 | interval updating Configuration data(millisecond) |
+| processInterval | N | 100 | interval processing data(millisecond) |
+| metricDomains | N | Sort | domain name of metric |
+| metricDomains.Sort.domainListeners | N | org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener | class name list of metric listener, separated by space |
+| prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
+| metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric data(millisecond) |
## SortClusterConfig
-- Get SortClusterConfig from the file:SortClusterConfig.conf in classpath, but it can not support online updating.
-- Get SortClusterConfig from InlongManager URL, but it can support online updating.
+The SortClusterConfig can be acquired from local file or manager, depending on the ***sortClusterConfig.type*** configured in common.properties.
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterName | Y | NA | inlong-sort-standalone cluster id |
-|sortTasks | Y | NA | Sort task list |
+| Parameter | Required | DefaultValue | Remark |
+|-------------|-------------|----------------|-----------------------------------|
+| clusterName | Y | NA | inlong-sort-standalone cluster id |
+| sortTasks | Y | NA | Sort task list |
+
+### SortTask
+Sort Task includes ***idParams and sinkParams*** which represent the config of ***stream and dataNode*** respectively
+
+The idParams and sinkParams are totally different among each type of sort tasks. There are two examples of hive task and pulsar task
+
+| Parameter | Required | DefaultValue | Remark |
+|------------|------------|----------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| name | Y | NA | sort task name |
+| type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n") |
+| idParams | Y | NA | List of Inlong DataStream configuration |
+| sinkParams | Y | NA | Sort task parameters |
-### SortTaskConfig
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|name | Y | NA | sort task name |
-|type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n") |
-|idParams | Y | NA | Inlong DataStream configuration |
-|sinkParams | Y | NA | Sort task parameters |
### idParams content of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|separator | Y | NA | separtor |
-|partitionIntervalMs | N | 3600000 | partition interval(millisecond) |
-|idRootPath | Y | NA | HDFS root path of Inlong DataStream |
-|partitionSubPath | Y | NA | partition sub path of Inlong DataStream |
-|hiveTableName | Y | NA | Hive table name of Inlong DataStream |
-|partitionFieldName | N | dt | partition field name of Inlong DataStream |
-|partitionFieldPattern | Y | NA | Date format of partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
-|msgTimeFieldPattern | Y | NA | Date format of message generation time, it support Java date format |
-|maxPartitionOpenDelayHour | N | 8 | Max delay hour of partition(hour) |
+| Parameter | Required | DefaultValue | Remark |
+|---------------------------|------------|---------------|--------------------------------------------------------------------------------------------|
+| inlongGroupId | Y | NA | inlongGroupId |
+| inlongStreamId | Y | NA | inlongStreamId |
+| separator | Y | NA | separator |
+| partitionIntervalMs | N | 3600000 | partition interval(millisecond) |
+| idRootPath | Y | NA | HDFS root path of Inlong DataStream |
+| partitionSubPath | Y | NA | partition sub path of Inlong DataStream |
+| hiveTableName | Y | NA | Hive table name of Inlong DataStream |
+| partitionFieldName | N | dt | partition field name of Inlong DataStream |
+| partitionFieldPattern | Y | NA | Date format of partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
+| msgTimeFieldPattern | Y | NA | Date format of message generation time, it support Java date format |
+| maxPartitionOpenDelayHour | N | 8 | Max delay hour of partition(hour) |
### sinkParams content of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|hdfsPath | Y | NA | NameNode URL of HDFS |
-|maxFileOpenDelayMinute | N | 5 | Max writing delay minute of simple HDFS file(minute) |
-|tokenOvertimeMinute | N | 60 | token overtime of Inlong Data Stream(minute) |
-|maxOutputFileSizeGb | N | 2 | Max file size of simple HDFS file(GB) |
-|hiveJdbcUrl | Y | NA | JDBC URL of Hive |
-|hiveDatabase | Y | NA | Hive database |
-|hiveUsername | Y | NA | Hive username |
-|hivePassword | Y | NA | Hive password |
+| Parameter | Required | DefaultValue | Remark |
+|------------------------|------------|-----------------|------------------------------------------------------|
+| hdfsPath | Y | NA | NameNode URL of HDFS |
+| maxFileOpenDelayMinute | N | 5 | Max writing delay minute of simple HDFS file(minute) |
+| tokenOvertimeMinute | N | 60 | token overtime of Inlong Data Stream(minute) |
+| maxOutputFileSizeGb | N | 2 | Max file size of simple HDFS file(GB) |
+| hiveJdbcUrl | Y | NA | JDBC URL of Hive |
+| hiveDatabase | Y | NA | Hive database |
+| hiveUsername | Y | NA | Hive username |
+| hivePassword | Y | NA | Hive password |
### idParams content of Pulsar sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|topic | Y | NA | Pulsarçš„Topic |
+| Parameter | Required | DefaultValue | Remark |
+|----------------|------------|-----------------|----------------|
+| inlongGroupId | Y | NA | inlongGroupId |
+| inlongStreamId | Y | NA | inlongStreamId |
+| topic | Y | NA | Topic of MQ |
### sinkParams content of Pulsar sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|serviceUrl | Y | NA | Pulsar service URL |
-|authentication | Y | NA | Pulsar authentication |
-|enableBatching | N | true | enableBatching |
-|batchingMaxBytes | N | 5242880 | batchingMaxBytes |
-|batchingMaxMessages | N | 3000 | batchingMaxMessages |
-|batchingMaxPublishDelay | N | 1 | batchingMaxPublishDelay |
-|maxPendingMessages | N | 1000 | maxPendingMessages |
-|maxPendingMessagesAcrossPartitions | N | 50000 | maxPendingMessagesAcrossPartitions |
-|sendTimeout | N | 0 | sendTimeout |
-|compressionType | N | NONE | compressionType |
-|blockIfQueueFull | N | true | blockIfQueueFull |
-|roundRobinRouterBatchingPartitionSwitchFrequency | N | 10 | roundRobinRouterBatchingPartitionSwitchFrequency |
+| Parameter | Required | DefaultValue | Remark |
+|--------------------------------------------------|-------------|----------------|--------------------------------------------------|
+| serviceUrl | Y | NA | Pulsar service URL |
+| authentication | Y | NA | Pulsar authentication |
+| enableBatching | N | true | enableBatching |
+| batchingMaxBytes | N | 5242880 | batchingMaxBytes |
+| batchingMaxMessages | N | 3000 | batchingMaxMessages |
+| batchingMaxPublishDelay | N | 1 | batchingMaxPublishDelay |
+| maxPendingMessages | N | 1000 | maxPendingMessages |
+| maxPendingMessagesAcrossPartitions | N | 50000 | maxPendingMessagesAcrossPartitions |
+| sendTimeout | N | 0 | sendTimeout |
+| compressionType | N | NONE | compressionType |
+| blockIfQueueFull | N | true | blockIfQueueFull |
+| roundRobinRouterBatchingPartitionSwitchFrequency | N | 10 | roundRobinRouterBatchingPartitionSwitchFrequency |
-### Sample of Hive sort task
-```
-{
- "data":{
- "clusterName":"hivev3-sz-sz1",
- "sortTasks":[
- {
- "idParams":[
- {
- "inlongGroupId":"0fc00000046",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
- "idRootPath":"/user/hive/warehouse/t_inlong_v1_0fc00000046",
- "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName":"t_inlong_v1_0fc00000046",
- "partitionFieldName":"dt",
- "partitionFieldPattern":"yyyyMMddHH",
- "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"03600000045",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
- "idRootPath":"/user/hive/warehouse/t_inlong_v1_03600000045",
- "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName":"t_inlong_v1_03600000045",
- "partitionFieldName":"dt",
- "partitionFieldPattern":"yyyyMMddHH",
- "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"05100054990",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
- "idRootPath":"/user/hive/warehouse/t_inlong_v1_05100054990",
- "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName":"t_inlong_v1_05100054990",
- "partitionFieldName":"dt",
- "partitionFieldPattern":"yyyyMMddHH",
- "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"09c00014434",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
- "idRootPath":"/user/hive/warehouse/t_inlong_v1_09c00014434",
- "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName":"t_inlong_v1_09c00014434",
- "partitionFieldName":"dt",
- "partitionFieldPattern":"yyyyMMddHH",
- "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"0c900035509",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
- "idRootPath":"/user/hive/warehouse/t_inlong_v1_0c900035509",
- "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName":"t_inlong_v1_0c900035509",
- "partitionFieldName":"dt",
- "partitionFieldPattern":"yyyyMMddHH",
- "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- }
- ],
- "name":"sid_hive_inlong6th_v3",
- "sinkParams":{
- "hdfsPath":"hdfs://127.0.0.1:9000",
- "maxFileOpenDelayMinute":"5",
- "tokenOvertimeMinute":"60",
- "maxOutputFileSizeGb":"2",
- "hiveJdbcUrl":"jdbc:hive2://127.0.0.2:10000",
- "hiveDatabase":"default",
- "hiveUsername":"hive",
- "hivePassword":"hive"
- },
- "type":"HIVE"
- }
- ]
- },
- "errCode":0,
- "md5":"md5",
- "result":true
-}
-```