You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by al...@apache.org on 2024/05/08 08:17:07 UTC
(inlong) branch dev-offline-sync updated (ebbbde7602 -> dadc2d194e)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
discard ebbbde7602 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
omit 1323736a0b [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
omit 5cdd94d4c9 [INLONG-9813][Manager] Support offline data sync management (#9814)
omit 875e07cd74 [INLONG-9781][Manager] Add offline sync task type definition (#9787)
add c727f8834a [INLONG-9837][Dashboard] Fix displays errors for Pulsar cluster management (#9838)
add fe982bc2b8 [INLONG-9820][Dashboard][Manager] Update Pulsar source field for Ingestion (#9821)
add b82f908cbf [INLONG-9753][Sort] Module inlong-sort-standalone's conf/sid_es_v3.conf file does not support adding Apache license (#9841)
add 69278edab8 [INLONG-9842][Dashboard] Data access resource details support paging (#9843)
add 9853017ccf [INLONG-9839][Manager] Optimize the auto assign logic of SortStandalone cluster (#9840)
add 7e1bff9970 [INLONG-9846][Manager] Optimize slow query SQL (#9847)
add 4f9f405984 [INLONG-9848][Agent] Add the getAuthHeader function in the HttpManager class (#9849)
add 7ab58db055 [INLONG-9844][Agent] Add agent installer config request class (#9845)
add 1731f2eecc [INLONG-9854][Sort] Fix the rowdata-tlogkv module artifactId (#9855)
add c5acb39654 [INLONG-9852][Agent] Place the configuration item for the installation package storage path in the installation package configuration (#9853)
add 9d4964e0d4 [INLONG-9850][Agent] Add a function to retrieve HttpManager in the ModuleManager class (#9851)
add f2dd0d6766 [INLONG-8948][Sort] Add Redis connector on flink 1.15 (#9836)
add 62c89a0acd [INLONG-9856][Manager] Fix the problem for missing tenant information when listTag (#9857)
add bc6a0ccc58 [INLONG-9860][Manager] Manager client supports querying workflow logs (#9861)
add c12be71518 [INLONG-9863][Agent] Avoiding data loss caused by too many supplementary files (#9864)
add 92794e15f3 [INLONG-9871][Sort] Use ZLIB as the default compression type of pulsar sink (#9872)
add 2f462677d5 [INLONG-9879][Sort] Sort inlongmsg-rowdata-kv format parse head has error (#9880)
add e46e411638 [INLONG-9876][Manager] optimize the format of error message (#9877)
add a68f4bcd34 [INLONG-9886][Manager] Optimized code DataProxyConfigRepositoryV2 (#9888)
add e42f5ec276 [INLONG-9873][Manager] Support adding data add tasks for file collection (#9874)
add 79528ac5aa [INLONG-9889][Agent] Delete test code, it will be added back in unit testing (#9890)
add 88d3958e12 [INLONG-9858][Agent] Increase local read and write capabilities for module config (#9892)
add 6477e99f8e [INLONG-9884][Manager] Optimized code AuditServiceImpl (#9896)
add 5bc955ded3 [INLONG-9895][Dashboard] Support restart when the task fails (#9898)
add c62bfff4ed [INLONG-9902][Manager] Data preview supports pulsar multi cluster (#9903)
add 1debef7ae0 [INLONG-9859][Agent] Increase installation package download capability and local MD5 computing power (#9901)
add 5b5eabbbea [INLONG-9897][Common] Code cognitive complexity modification (#9900)
add 595bb12d75 [INLONG-9893][Common] Optimized code InLongMsg (#9894)
add 093333de40 [INLONG-9906][Agent] Add configuration comparison logic and processing of comparison results (#9908)
add f756ccc7ba [INLONG-9910][Agent] Increase daily verification for process monitoring, reinstallation, or re downloading (#9912)
add 18b0bf686f [INLONG-9904][Audit] SDK support checkpoint feature for Flink job (#9905)
add 3746e3ffcc [INLONG-9907][Audit] Audit-service add codes of entities (#9911)
add 8903184b2a [INLONG-9866][Sort] Add end to end test case(redis to redis) for sort-connector-redis-v1.15. (#9869)
add 73c9350011 [INLONG-9914][Audit] Add SQL related to audit-service (#9915)
add 30dd38f8af [INLONG-9909][Agent] Add unit test for installer (#9916)
add 8dcf31a5ad [INLONG-9918][Audit] Audit-service add codes of config (#9919)
add b4f099ada2 [INLONG-9930] Add pom for audit-service (#9931)
add 03e8d71817 [INLONG-9913][Sort] Solve end-to-end-tests-v1.13 naming problem and test case run failed. (#9924)
add 49932d6f59 [INLONG-9922][Agent] Add a script to copy the configuration from the installer to the agent (#9923)
add 9b5999764a [INLONG-9926][Audit] Audit-service support HA active and backup (#9927)
add 1df84dca9f [INLONG-9934][doc] update copyright date to 2024 (#9935)
add da856a4619 [INLONG-9928][Audit] Audit-service HA election through mysql (#9929)
add e3543ccea8 [INLONG-9938][Dashboard] Support Agent module and package page (#9939)
add f2263cea7a [INLONG-9932][Manager] Add an agent installer module management for agent installation (#9933)
add 84bd8b247d [INLONG-9936][Manager] Support for parallel acquisition of data preview data from multiple Pulsar clusters (#9937)
add da4c77e2cd [INLONG-9920][Audit] Audit-service add codes of source (#9941)
add f5950eed5a [INLONG-9921][Manager] Fix the problem of manager can't stop data sync job (#9942)
add bcc1f3fd9f [INLONG-9946][Agent] Verify the return code, only proceed with the subsequent process if the return code is successful (#9947)
add 50539a5660 [INLONG-9943][Audit] Audit-service add codes of jdbc sink (#9949)
add c13e35528e [INLONG-9953][Manager] Fix the problem of stop stream source failed (#9954)
add c9da983405 [INLONG-9950][Dashboard] Fix audit query using incorrect end time (#9951)
add 98c84917a3 [INLONG-9944][Dashboard] Add Agent and Installer for Agent Cluster Node (#9945)
add ec6ee225d4 [INLONG-9948][Agent] Optimize the instance class to decrease the complexity of usage (#9952)
add d6f03f3c17 [INLONG-9955][Agent] Rename Job to Task for agent (#9956)
add 25f7bafa4c [INLONG-9402][Sort] Solve errors in UT test logs for sort-end-to-end-tests, but UT results show as successful (#9940)
add 80ec84e9c7 [INLONG-9957][Audit] Audit-service add local cache for openapi (#9958)
add 9c64aefb2d [INLONG-9965][Sort] Fix wrong node duration of SortStandalone Pulsar Sink (#9966)
add dba59007b6 [INLONG-9962][Manager] Data preview supports returning header and specific field information (#9967)
add bff9c2cfb5 [INLONG-9968][Manager] Support pulsar multi cluster when creating pulsar consumption groups (#9970)
add 24bdfc1f9e [INLONG-9969][Agent] Release the memory semaphore of the source only when the data is placed in the queue (#9971)
add 90890c2dd4 [INLONG-9870][Manager] Pulsar DataNode support to set compression type (#9959)
add d91262e261 [INLONG-9963][Dashboard][Agent] Add agentIp fields to Kafka and Pulsar source (#9964)
add 99d879207d [INLONG-9974][Manager] Data preview simplifies interface field information (#9975)
add 7e0e016160 [INLONG-9976][Manager] Support multiple types of audit indicator queries (#9979)
add 3d4716983b [INLONG-9977][Audit] Audit-service increases the capabilities of openapi (#9978)
add bf8d1f7f40 [INLONG-9925][Sort] Add if statement to solve continuously printing logs (#9961)
add b1de06b94e [INLONG-9972][Sort] Pulsar connector support authentication when connecting to Pulsar cluster (#9973)
add 4ebe63fbc2 [INLONG-9985][Manager] Support authentication params for pulsar source (#9986)
add 0fee4a1fa4 [INLONG-9980][Manager] Remove the derby.jar file from the manager project (#9981)
add 5b559ee139 [INLONG-9987][Agent] Fix the issue of deleting the first digit when the first digit in MD5 is 0 (#9988)
add 62129b7e43 [INLONG-9990][Agent] Avoid MD5 calculation functions directly returning (#9991)
add 967a21afae [INLONG-9989][Audit] Audit-store support the feature of audit-version (#9992)
add d6c523c256 [INLONG-9993][Audit] The data protocol adds an audit_version attribute (#9994)
add 6b5ab82ce0 [INLONG-9982][Agent] Adjusting the abstraction of source code to facilitate rapid addition of sources (#9984)
add c8c050e333 [INLONG-9997][Agent] Handling situations where the installation package md5 remains unchanged and other parameters md5 change (#9998)
add 5d7e124461 [INLONG-10001][Dashboard] Fix end date time initialization error on audit page (#10002)
add a6d5fae18b [INLONG-9999][Agent] Handle scenarios where the module list is empty to prevent accidental deletion (#10000)
add b5b3962cde [INLONG-10003][Audit] Audit-service supports multiple data source clusters (#10004)
add 9636f16704 [INLONG-10006][Agent] Add MongoDB data source for Agent (#10007)
add 95352390c1 [INLONG-9995][Manager] Support batch saving of group information and other operations (#9996)
add 4152f98df9 [INLONG-10014][Audit] Add new audit SDK API for Sort Flink
add f4db5af56c [INLONG-9875][Sort] Remove the repeated DorisLoadNode (#10015)
add b206b830a1 [INLONG-10012][Agent] Adjust task encapsulation to place common logicin the base class (#10013)
add a3cc5011f8 [INLONG-10010][Agent] Adjust source encapsulation, keep public initialization in the base class init, and place specific source related initialization in the subclass's initSource (#10011)
add 27d577702c [INLONG-10008][Dashboard][Manager] Supplement fields of MongoDB data source for Dashboard (#10009)
add fbf1ffcb69 [INLONG-9960][Manager] Support to config kafka data node (#10005)
add 973974ebde [INLONG-8962][Agent] Fix the bug: before state and after state are the same in StateCallback.call (#10021)
add 51420af87a [INLONG-9883][Manager] Add unit test of HeartbeatController (#10019)
add 33b4ca9db4 [INLONG-10017][Sort] Add the definition of Sort Config (#10018)
add 8a7568e5bc [INLONG-10024][Release] Add the 1.12.0 version option for the bug report (#10036)
add 99f1b8d875 [INLONG-10034][Manager] Support calling the audit service interface to query audit information (#10037)
add fc5ce9dde7 [INLONG-10025][Release] Update changes log for the 1.12.0 version (#10043)
add cd2a7a84a0 [INLONG-10035][Agent] Report audit data using the new SDK interface (#10038)
add 05b92ddbed [INLONG-8791][TubeMQ] Add log level and log path configuration API to Tubemq-client-go (#10040)
add 0bee32f0ee [INLONG-10027][Release] Bumped master branch version to 1.13.0-SNAPSHOT (#10033)
add 255be6052e [INLONG-10045][Dashboard] Supports one approval order containing multiple inlongGroups (#10046)
add 3abb856200 [INLONG-10041][Sort] Thread unsafety of time format serialization (#10042)
add fbd1e531c8 [INLONG-10047][Agent] Fix the bug in the logic for judging the completion status of the supplementary recording task (#10050)
add b3b505f5ee [INLONG-10048][Tools] Remove more useless paths to release more size for workflows (#10049)
add 4c04f3af33 [INLONG-10051][CI] Improve the pull request template (#10052)
add 6eada3b702 [INLONG-10054][Sort] Add dependencies for flink 1.18 (#10058)
add 8be68c58a9 [INLONG-10060][Manager] Optimize audit data query related logs (#10061)
add 5e836457cc [INLONG-10059][Agent] Fix the exception of installation package comparison (#10062)
add 4db62df903 [INLONG-10053][Sort] Support flink-connector-pulsar based on flink 1.18 (#10063)
add 5a0ae62b94 [INLONG-10067][DataProxy] Simplify the configuration and acquisition of the Manager address (#10068)
add 13771b08fd [INLONG-10071][Manager] Supplementary the SQL statement trailing semicolon in apache_inlong_manager.sql (#10072)
add 721d61a77a [INLONG-10074][Manager] Fix the problem of Error in querying audit information based on IP address (#10075)
add b0357fa163 [INLONG-10056][Manager] Support new manager plugin for flink 1.18 (#10077)
add 60c2676223 [INLONG-10066][DataProxy] Optimize the related configuration settings in the CommonConfigHolder.java file (#10079)
add b5714f8e26 [INLONG-10081][DataProxy] Modify the data format of metadata saved in the metadata.json file (#10083)
add 40cc17d170 [INLONG-10082][Dashboard] Fixed missing pulsar tenant field when updating group info (#10084)
add f2110a4316 [INLONG-10055][Sort] Support flink-connector-jdbc based on flink 1.18 (#10073)
add 5674bc24f8 [INLONG-10069][Sort] Support audit metrics for sort-connector-pulsar-1.18 (#10070)
add b408d7f166 [INLONG-10086][Dashboard] Add refresh function of module audit (#10087)
add 7012648891 [INLONG-10089][Audit] Adjust the service configuration of Audit (#10090)
add a2e0f464dc [INLONG-10085][Audit] Optimize the performance of the audit-service (#10088)
add fabb0c466b [INLONG-10094][agent] Fix the bug: the task record for data supplementation has not expired (#10095)
add eba89c79bd [INLONG-10102][DataProxy] Adjust the position where Source calls addSourceReportInfo() (#10103)
add d963e5c753 [INLONG-10099][Audit] Audit-store adds the general JDBC sink capability (#10104)
add ec1740e3ca [INLONG-10093][SDK] Support to transform from CSV protocol to CSV/KV protocol by single SQL (#10097)
add 28bfa89e54 [INLONG-10017][Agent] Fix the module update configuration bug (#10108)
add 364bdc5a24 [INLONG-10111][DataProxy] Add auditVersion field processing (#10112)
add 6260e1a705 [INLONG-10113][Agent] Delete useless code (#10114)
add 2e70a12a2f [Manager] Fix sort standalone get kafka config error (#10106)
add ba475c15c4 [INLONG-10115][Agent] Change the offset save format from long to string (#10116)
add 22d6859e2f [INLONG-9867][Manager] Unified configuration process for standalone and sortflink (#9868)
add 837e5ad0db [INLONG-10120][CI] Update the actions/upload-artifact to V4 (#10121)
add b279bc84ac [INLONG-10122][Audit] Update the apache_inlong_audit.sql to apache_inlong_audit_mysql.sql (#10123)
add 714d7af5b6 [INLONG-10109][SDK] Support to transform from Json protocol to CSV/KV protocol by single SQL (#10110)
add c4cac2e981 [INLONG-10117][SDK] Support to transform from PB protocol to CSV/KV protocol by single SQL (#10127)
add 2deb3d33e9 [INLONG-10126][TubeMQ] Fix maven sleepycat je.version 7.3.7 can not found exception (#10131)
add ae49243a5a [INLONG-10129][SDK] Transform SQL support +-*/ operations (#10133)
add dd9aa7abc4 [INLONG-10136][Manager] Clean up ElasticSearch and ClickHouse audit source query function (#10140)
add a88dd9db26 [INLONG-10138][CI] Bump actions/checkout@v3, actions/setup-java@v3, actions/cache@v3 to v4 (#10139)
add 585b9fb033 [INLONG-10141][Manager] Data preview supports returning header and specific field information for inlong msg v1 (#10142)
add 9c40c95e72 [INLONG-10130][SDK] Transform SQL support string concat function (#10143)
add 3c703dd1d3 [INLONG-10135][Manager] Move inlongCompressType to clustertag configuration (#10137)
add aa674dc79d [INLONG-10132][Audit] Clean up ElasticSearch and ClickHouse related code of audit-store (#10134)
add bf42e1a5f8 [INLONG-10091][Sort] Add json format for Flink 1.18 (#10092)
new cfd9136286 [INLONG-9781][Manager] Add offline sync task type definition (#9787)
new e66cf075fe [INLONG-9813][Manager] Support offline data sync management (#9814)
new ddc72ed417 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
new dadc2d194e [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ebbbde7602)
\
N -- N -- N refs/heads/dev-offline-sync (dadc2d194e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ISSUE_TEMPLATE/bug-report.yml | 8 +-
.github/PULL_REQUEST_TEMPLATE.md | 17 +-
.github/workflows/ci_build.yml | 53 +-
.github/workflows/ci_chart_test.yml | 2 +-
.github/workflows/ci_check_format.yml | 4 +-
.github/workflows/ci_check_license.yml | 2 +-
.github/workflows/ci_check_pr_title.yml | 2 +-
.github/workflows/ci_docker.yml | 40 +-
.github/workflows/ci_ut.yml | 12 +-
.github/workflows/ci_ut_flink13.yml | 96 ++++
.github/workflows/ci_ut_flink15.yml | 24 +-
.github/workflows/codeql_analysis.yml | 40 +-
CHANGES.md | 262 +++++----
NOTICE | 2 +-
bin/inlong-daemon | 2 +-
docker/docker-compose/README.md | 2 +-
inlong-agent/agent-common/pom.xml | 2 +-
.../apache/inlong/agent/conf/InstanceProfile.java | 8 +-
.../apache/inlong/agent/conf/OffsetProfile.java | 10 +-
.../inlong/agent/constant/AgentConstants.java | 65 ---
.../inlong/agent/constant/CommonConstants.java | 70 ---
.../inlong/agent/constant/FetcherConstants.java | 3 -
.../apache/inlong/agent/constant/JobConstants.java | 161 ------
.../inlong/agent/constant/KubernetesConstants.java | 36 --
.../inlong/agent/constant/MetadataConstants.java | 41 --
.../inlong/agent/constant/TaskConstants.java | 195 ++-----
.../org/apache/inlong/agent/db/InstanceDb.java | 14 +
.../org/apache/inlong/agent/db/JobProfileDb.java | 253 ---------
.../inlong/agent/message/file/OffsetAckInfo.java | 2 +-
.../inlong/agent/message/file/ProxyMessage.java | 3 +-
.../agent/message/file/ProxyMessageCache.java | 2 +
.../inlong/agent/metrics/audit/AuditUtils.java | 16 +-
.../apache/inlong/agent/plugin/file/Source.java | 6 +
.../org/apache/inlong/agent/pojo/BinlogJob.java | 83 ---
.../org/apache/inlong/agent/pojo/BinlogTask.java | 83 +++
.../org/apache/inlong/agent/pojo/FileTask.java | 17 -
.../org/apache/inlong/agent/pojo/KafkaJob.java | 76 ---
.../org/apache/inlong/agent/pojo/KafkaTask.java | 76 +++
.../org/apache/inlong/agent/pojo/MongoJob.java | 118 ----
.../org/apache/inlong/agent/pojo/MongoTask.java | 85 +++
.../java/org/apache/inlong/agent/pojo/MqttJob.java | 57 --
.../org/apache/inlong/agent/pojo/MqttTask.java | 57 ++
.../org/apache/inlong/agent/pojo/OracleJob.java | 75 ---
.../org/apache/inlong/agent/pojo/OracleTask.java | 75 +++
.../apache/inlong/agent/pojo/PostgreSQLJob.java | 58 --
.../apache/inlong/agent/pojo/PostgreSQLTask.java | 58 ++
.../org/apache/inlong/agent/pojo/PulsarTask.java | 6 +-
.../org/apache/inlong/agent/pojo/RedisJob.java | 46 --
.../org/apache/inlong/agent/pojo/RedisTask.java | 46 ++
.../org/apache/inlong/agent/pojo/SqlServerJob.java | 76 ---
.../apache/inlong/agent/pojo/SqlServerTask.java | 76 +++
.../apache/inlong/agent/pojo/TaskProfileDto.java | 408 +++++++-------
.../inlong/agent/state/AbstractStateWrapper.java | 3 +-
.../org/apache/inlong/agent/utils/HttpManager.java | 21 +-
inlong-agent/agent-core/pom.xml | 2 +-
.../agent/core/instance/InstanceManager.java | 75 +--
.../inlong/agent/core/task/OffsetManager.java | 93 ++-
.../apache/inlong/agent/core/task/TaskManager.java | 2 +-
inlong-agent/agent-docker/pom.xml | 2 +-
.../agent-installer/conf/installer.properties | 12 +-
inlong-agent/agent-installer/pom.xml | 20 +-
.../inlong/agent/installer/ManagerFetcher.java | 116 +---
.../inlong/agent/installer/ModuleManager.java | 498 +++++++++++++++-
.../src/test/java/installer/BaseTestsHelper.java | 67 +++
.../test/java/installer/ModuleActionTypeEnum.java | 57 ++
.../src/test/java/installer/TestModuleManager.java | 208 +++++++
.../src/test/resources/conf/modules.json | 1 +
.../agent-installer/src/test/resources/log4j2.xml | 45 ++
inlong-agent/agent-plugins/pom.xml | 2 +-
.../agent/plugin/fetcher/ManagerFetcher.java | 6 +-
.../agent/plugin/instance/CommonInstance.java | 206 +++++++
.../inlong/agent/plugin/instance/FileInstance.java | 174 +-----
.../agent/plugin/instance/KafkaInstance.java | 170 +-----
.../agent/plugin/instance/MongoDBInstance.java | 29 +
.../agent/plugin/instance/PulsarInstance.java | 162 +-----
.../plugin/sinks/filecollect/SenderManager.java | 28 +-
.../agent/plugin/sources/DatabaseSqlSource.java | 36 ++
.../inlong/agent/plugin/sources/KafkaSource.java | 302 +++-------
.../inlong/agent/plugin/sources/LogFileSource.java | 450 +++------------
.../inlong/agent/plugin/sources/MongoDBSource.java | 157 +++++-
.../inlong/agent/plugin/sources/MqttSource.java | 36 ++
.../inlong/agent/plugin/sources/OracleSource.java | 36 ++
.../agent/plugin/sources/PostgreSQLSource.java | 36 ++
.../inlong/agent/plugin/sources/PulsarSource.java | 281 +++------
.../inlong/agent/plugin/sources/RedisSource.java | 36 ++
.../agent/plugin/sources/SQLServerSource.java | 36 ++
.../agent/plugin/sources/file/AbstractSource.java | 347 ++++++++++++
.../agent/plugin/sources/reader/MongoDBReader.java | 122 ++--
.../reader/file/KubernetesMetadataProvider.java | 121 ----
.../inlong/agent/plugin/task/AbstractTask.java | 159 ++++++
.../apache/inlong/agent/plugin/task/KafkaTask.java | 128 +----
.../inlong/agent/plugin/task/MongoDBTask.java | 71 +++
.../inlong/agent/plugin/task/PulsarTask.java | 121 +---
.../inlong/agent/plugin/task/file/LogFileTask.java | 142 ++---
.../inlong/agent/plugin/utils/MetaDataUtils.java | 139 -----
.../inlong/agent/plugin/utils/PluginUtils.java | 39 +-
.../inlong/agent/plugin/utils/RocksDBUtils.java | 6 -
.../agent/plugin/utils/file/FileDataUtils.java | 122 ----
.../apache/inlong/agent/plugin/sinks/MockSink.java | 8 +-
.../sinks/filecollect/TestSenderManager.java | 2 +-
.../agent/plugin/sources/TestLogFileSource.java | 9 +-
.../agent/plugin/utils/MetaDataUtilsTest.java | 37 --
inlong-agent/agent-release/pom.xml | 2 +-
inlong-agent/bin/agent-config.sh | 70 +++
inlong-agent/conf/agent.properties | 2 +-
inlong-agent/pom.xml | 2 +-
inlong-audit/audit-common/pom.xml | 2 +-
.../inlong/audit/consts/ConfigConstants.java | 1 +
inlong-audit/audit-docker/Dockerfile | 22 +-
inlong-audit/audit-docker/audit-docker.sh | 63 ++-
inlong-audit/audit-docker/pom.xml | 2 +-
inlong-audit/audit-proxy/pom.xml | 2 +-
inlong-audit/audit-release/pom.xml | 7 +-
inlong-audit/audit-sdk/README.md | 76 ++-
inlong-audit/audit-sdk/pom.xml | 2 +-
.../org/apache/inlong/audit/AuditReporterImpl.java | 325 ++++++++---
.../apache/inlong/audit/send/SenderManager.java | 37 +-
.../apache/inlong/audit/util/AuditDimensions.java | 99 ++++
.../org/apache/inlong/audit/util/AuditValues.java | 58 ++
inlong-audit/audit-service/pom.xml | 306 ++++++++++
.../apache/inlong/audit/cache/AbstractCache.java | 152 +++++
.../org/apache/inlong/audit/cache/DayCache.java | 160 ++++++
.../apache/inlong/audit/cache/HalfHourCache.java | 44 ++
.../org/apache/inlong/audit/cache/HourCache.java | 42 ++
.../apache/inlong/audit/cache/RealTimeQuery.java | 361 ++++++++++++
.../apache/inlong/audit/cache/TenMinutesCache.java | 43 ++
.../org/apache/inlong/audit/channel/DataQueue.java | 70 +++
.../inlong/audit/config/ConfigConstants.java | 105 ++++
.../apache/inlong/audit/config/Configuration.java | 110 ++++
.../inlong/audit/config/OpenApiConstants.java | 68 +++
.../apache/inlong/audit/config/SqlConstants.java | 149 +++++
.../org/apache/inlong/audit/entities/ApiType.java | 25 +
.../apache/inlong/audit/entities/AuditCycle.java | 51 ++
.../apache/inlong/audit/entities/JdbcConfig.java | 31 +
.../org/apache/inlong/audit/entities/Request.java | 31 +
.../apache/inlong/audit/entities/SinkConfig.java | 36 ++
.../apache/inlong/audit/entities/SourceConfig.java | 54 ++
.../apache/inlong/audit/entities/StartEndTime.java | 28 +
.../org/apache/inlong/audit/entities/StatData.java | 39 ++
.../org/apache/inlong/audit/main/Application.java | 127 +++++
.../audit/selector/SelectorChangeListenerImpl.java | 35 ++
.../apache/inlong/audit/selector/api/Selector.java | 42 ++
.../audit/selector/api/SelectorChangeListener.java | 26 +
.../inlong/audit/selector/api/SelectorConfig.java | 88 +++
.../inlong/audit/selector/api/SelectorFactory.java | 30 +
.../inlong/audit/selector/impl/DBDataSource.java | 291 ++++++++++
.../inlong/audit/selector/impl/SelectorImpl.java | 206 +++++++
.../inlong/audit/selector/task/DBMonitorTask.java | 63 +++
.../apache/inlong/audit/service/ApiService.java | 311 ++++++++++
.../apache/inlong/audit/service/ConfigService.java | 228 ++++++++
.../apache/inlong/audit/service/EtlService.java | 244 ++++++++
.../org/apache/inlong/audit/sink/CacheSink.java | 90 +++
.../org/apache/inlong/audit/sink/JdbcSink.java | 164 ++++++
.../org/apache/inlong/audit/source/JdbcSource.java | 319 +++++++++++
.../org/apache/inlong/audit/utils/CacheUtils.java | 35 ++
.../org/apache/inlong/audit/utils/JdbcUtils.java | 67 +++
inlong-audit/audit-store/pom.xml | 2 +-
.../inlong/audit/config/ClickHouseConfig.java | 50 --
.../inlong/audit/config/DataServerConfig.java | 47 --
.../inlong/audit/config/ElasticsearchConfig.java | 125 ----
.../org/apache/inlong/audit/config/JdbcConfig.java | 44 ++
.../apache/inlong/audit/config/StoreConfig.java | 16 +-
.../org/apache/inlong/audit/db/DruidConfig.java | 80 ---
.../inlong/audit/db/DruidDataSourceProperties.java | 47 --
.../apache/inlong/audit/db/dao/AuditDataDao.java | 28 -
.../inlong/audit/db/entities/AuditDataPo.java | 45 --
.../inlong/audit/db/entities/ClickHouseDataPo.java | 44 --
.../apache/inlong/audit/db/entities/ESDataPo.java | 48 --
.../inlong/audit/db/entities/JdbcDataPo.java | 46 ++
.../audit/service/AuditMsgConsumerServer.java | 39 +-
.../inlong/audit/service/ClickHouseService.java | 204 -------
.../inlong/audit/service/ElasticsearchService.java | 349 ------------
.../apache/inlong/audit/service/InsertData.java | 5 +
.../apache/inlong/audit/service/JdbcService.java | 224 ++++++++
.../apache/inlong/audit/service/MySqlService.java | 56 --
.../inlong/audit/service/consume/BaseConsume.java | 13 +-
.../audit/service/consume/PulsarConsume.java | 3 +-
.../src/main/resources/mapper/AuditDataDao.xml | 48 --
.../src/main/resources/sql/apache_inlong_audit.sql | 51 --
.../sql/apache_inlong_audit_clickhouse.sql | 45 --
.../audit/service/ElasticsearchServiceTest.java | 96 ----
.../audit/service/consume/KafkaConsumeTest.java | 15 +-
.../audit/service/consume/TubeConsumeTest.java | 17 +-
.../src/test/resources/mapper/AuditDataDao.xml | 47 --
inlong-audit/bin/service-start.sh | 85 +++
inlong-audit/bin/service-stop.sh | 42 ++
inlong-audit/conf/application.properties | 60 +-
inlong-audit/conf/audit-proxy-kafka.conf | 8 +-
inlong-audit/conf/audit-proxy-pulsar.conf | 8 +-
inlong-audit/conf/audit-proxy-tubemq.conf | 8 +-
inlong-audit/conf/audit-service.properties | 24 +
inlong-audit/conf/mapper/AuditDataDao.xml | 47 --
inlong-audit/pom.xml | 14 +-
inlong-audit/sql/apache_inlong_audit.sql | 52 --
.../sql/apache_inlong_audit_clickhouse.sql | 46 --
inlong-audit/sql/apache_inlong_audit_mysql.sql | 134 +++++
inlong-audit/sql/apache_inlong_audit_starrocks.sql | 49 ++
inlong-audit/sql/clickhouse-changes-1.9.0.sql | 25 -
inlong-audit/sql/mysql-changes-1.9.0.sql | 25 -
inlong-common/pom.xml | 10 +-
.../inlong/common/constant/CompressionType.java | 26 +
.../apache/inlong/common/constant/Constants.java | 3 +
.../inlong/common/constant/DataNodeType.java | 26 +
.../common/constant/DeserializationType.java | 26 +
.../apache/inlong/common/constant/SinkType.java | 26 +
.../apache/inlong/common/enums/DataTypeEnum.java | 20 +-
.../apache/inlong/common/enums/IndicatorType.java | 68 +++
.../inlong/common/enums/InlongCompressType.java | 6 +
.../apache/inlong/common/monitor/CounterGroup.java | 6 +-
.../inlong/common/monitor/CounterGroupExt.java | 6 +-
.../inlong/common/msg/AttributeConstants.java | 5 +-
.../org/apache/inlong/common/msg/InLongMsg.java | 20 +-
.../inlong/common/msg/InLongMsgAttrBuilder.java | 57 +-
.../common/pojo/agent/installer/ConfigRequest.java | 35 ++
.../common/pojo/agent/installer/ConfigResult.java | 14 +-
.../common/pojo/agent/installer/InstallerCode.java | 56 ++
.../common/pojo/agent/installer/ModuleConfig.java | 13 +-
.../common/pojo/agent/installer/PackageConfig.java | 4 +
.../inlong/common/pojo/sort/SortClusterConfig.java | 36 ++
.../apache/inlong/common/pojo/sort/SortConfig.java | 30 +
.../inlong/common/pojo/sort/SortTaskConfig.java | 35 ++
.../common/pojo/sort/dataflow/DataFlowConfig.java | 37 ++
.../common/pojo/sort/dataflow/SourceConfig.java | 38 ++
.../pojo/sort/dataflow/dataType/CsvConfig.java | 27 +
.../sort/dataflow/dataType/DataTypeConfig.java | 34 ++
.../pojo/sort/dataflow/dataType/KvConfig.java | 28 +
.../deserialization/DeserializationConfig.java | 34 ++
.../InlongMsgDeserializationConfig.java | 26 +
.../InlongMsgPbDeserialiationConfig.java | 26 +
.../pojo/sort/dataflow/field/FieldConfig.java | 31 +
.../dataflow/field/format/ArrayFormatInfo.java | 83 +++
.../sort/dataflow/field/format/ArrayTypeInfo.java | 77 +++
.../dataflow/field/format/BasicFormatInfo.java | 42 ++
.../dataflow/field/format/BinaryFormatInfo.java | 96 ++++
.../sort/dataflow/field/format/BinaryTypeInfo.java | 45 ++
.../dataflow/field/format/BooleanFormatInfo.java | 62 ++
.../dataflow/field/format/BooleanTypeInfo.java | 47 ++
.../sort/dataflow/field/format/ByteFormatInfo.java | 62 ++
.../sort/dataflow/field/format/ByteTypeInfo.java | 47 ++
.../pojo/sort/dataflow/field/format/Constants.java | 26 +
.../sort/dataflow/field/format/DateFormatInfo.java | 145 +++++
.../sort/dataflow/field/format/DateTypeInfo.java | 47 ++
.../dataflow/field/format/DecimalFormatInfo.java | 106 ++++
.../dataflow/field/format/DecimalTypeInfo.java | 47 ++
.../dataflow/field/format/DoubleFormatInfo.java | 62 ++
.../sort/dataflow/field/format/DoubleTypeInfo.java | 47 ++
.../dataflow/field/format/FloatFormatInfo.java | 62 ++
.../sort/dataflow/field/format/FloatTypeInfo.java | 47 ++
.../sort/dataflow/field/format/FormatInfo.java | 61 ++
.../sort/dataflow/field/format/FormatUtils.java | 75 +++
.../sort/dataflow/field/format/IntFormatInfo.java | 62 ++
.../sort/dataflow/field/format/IntTypeInfo.java | 47 ++
.../format/LocalZonedTimestampFormatInfo.java | 86 +++
.../field/format/LocalZonedTimestampTypeInfo.java | 45 ++
.../sort/dataflow/field/format/LongFormatInfo.java | 62 ++
.../sort/dataflow/field/format/LongTypeInfo.java | 47 ++
.../sort/dataflow/field/format/MapFormatInfo.java | 101 ++++
.../sort/dataflow/field/format/MapTypeInfo.java | 93 +++
.../sort/dataflow/field/format/NullFormatInfo.java | 50 ++
.../sort/dataflow/field/format/NullTypeInfo.java | 45 ++
.../sort/dataflow/field/format/RowFormatInfo.java | 123 ++++
.../sort/dataflow/field/format/RowTypeInfo.java | 146 +++++
.../dataflow/field/format/ShortFormatInfo.java | 62 ++
.../sort/dataflow/field/format/ShortTypeInfo.java | 47 ++
.../dataflow/field/format/StringFormatInfo.java | 62 ++
.../sort/dataflow/field/format/StringTypeInfo.java | 47 ++
.../sort/dataflow/field/format/TimeFormatInfo.java | 162 ++++++
.../sort/dataflow/field/format/TimeTypeInfo.java | 47 ++
.../dataflow/field/format/TimestampFormatInfo.java | 169 ++++++
.../dataflow/field/format/TimestampTypeInfo.java | 47 ++
.../pojo/sort/dataflow/field/format/TypeInfo.java | 51 ++
.../dataflow/field/format/VarBinaryFormatInfo.java | 111 ++++
.../dataflow/field/format/VarBinaryTypeInfo.java | 48 ++
.../dataflow/field/format/VarCharFormatInfo.java | 84 +++
.../pojo/sort/dataflow/sink/ClsSinkConfig.java | 36 ++
.../pojo/sort/dataflow/sink/EsSinkConfig.java | 29 +
.../pojo/sort/dataflow/sink/PulsarSinkConfig.java | 31 +
.../common/pojo/sort/dataflow/sink/SinkConfig.java | 41 ++
.../common/pojo/sort/mq/MqClusterConfig.java | 38 ++
.../common/pojo/sort/mq/PulsarClusterConfig.java | 33 ++
.../common/pojo/sort/mq/TubeClusterConfig.java | 28 +
.../common/pojo/sort/node/ClsNodeConfig.java | 34 ++
.../inlong/common/pojo/sort/node/EsNodeConfig.java | 38 ++
.../inlong/common/pojo/sort/node/NodeConfig.java | 39 ++
.../common/pojo/sort/node/PulsarNodeConfig.java | 29 +
.../pojo/sortstandalone/SortConfigResponse.java | 41 ++
inlong-dashboard/pom.xml | 2 +-
inlong-dashboard/src/configs/menus/conf.tsx | 8 +
inlong-dashboard/src/configs/routes/conf.ts | 8 +
inlong-dashboard/src/i18n.ts | 4 +
.../src/plugins/clusters/defaults/Pulsar.ts | 4 +-
.../plugins/sources/common/SourceDefaultInfo.ts | 1 +
.../src/plugins/sources/defaults/Kafka.ts | 94 ++-
.../src/plugins/sources/defaults/Mongodb.ts | 126 ++++-
.../src/plugins/sources/defaults/Pulsar.ts | 183 +++++-
inlong-dashboard/src/ui/locales/cn.json | 35 +-
inlong-dashboard/src/ui/locales/en.json | 35 +-
.../src/ui/pages/AgentModule/AgentModuleTag.tsx | 143 +++++
.../src/ui/pages/AgentModule/CreateModal.tsx | 189 +++++++
.../src/ui/pages/AgentModule/config.tsx | 76 +++
.../src/ui/pages/AgentModule/index.tsx | 82 +++
.../src/ui/pages/AgentPackage/AgentPackageTag.tsx | 143 +++++
.../src/ui/pages/AgentPackage/CreateModal.tsx | 123 ++++
.../src/ui/pages/AgentPackage/config.tsx | 80 +++
.../src/ui/pages/AgentPackage/index.tsx | 82 +++
.../src/ui/pages/Clusters/NodeEditModal.tsx | 76 +++
.../src/ui/pages/GroupDashboard/config.tsx | 2 +-
.../src/ui/pages/GroupDetail/Audit/config.tsx | 2 +-
.../src/ui/pages/GroupDetail/Audit/index.tsx | 2 +-
.../src/ui/pages/GroupDetail/Info/index.tsx | 3 +
.../ui/pages/GroupDetail/ResourceInfo/index.tsx | 70 ++-
.../src/ui/pages/ModuleAudit/IdModule/config.tsx | 11 +-
.../src/ui/pages/ModuleAudit/IdModule/index.tsx | 11 +-
.../src/ui/pages/ModuleAudit/IpModule/config.tsx | 11 +-
.../src/ui/pages/Process/Approvals/config.tsx | 12 +-
.../src/ui/pages/SynchronizeDashboard/config.tsx | 2 +-
.../src/ui/pages/SynchronizeDetail/Info/index.tsx | 3 +
inlong-dataproxy/conf/common.properties | 12 +-
inlong-dataproxy/dataproxy-dist/pom.xml | 2 +-
inlong-dataproxy/dataproxy-docker/pom.xml | 2 +-
inlong-dataproxy/dataproxy-source/pom.xml | 2 +-
.../dataproxy/config/CommonConfigHolder.java | 605 ++++++++++++--------
.../inlong/dataproxy/config/ConfigHolder.java | 4 +-
.../inlong/dataproxy/config/ConfigManager.java | 299 ++++++++--
.../config/DefaultManagerIpListParser.java | 65 ---
.../dataproxy/config/IManagerIpListParser.java | 35 --
.../dataproxy/config/holder/MetaConfigHolder.java | 372 +++++-------
.../config/holder/SourceReportConfigHolder.java | 7 +-
.../dataproxy/config/pojo/CacheClusterConfig.java | 20 +
.../inlong/dataproxy/config/pojo/CacheType.java | 11 +-
.../inlong/dataproxy/config/pojo/DataType.java | 70 ---
.../dataproxy/config/pojo/IdTopicConfig.java | 61 +-
.../dataproxy/config/pojo/InLongMetaConfig.java | 69 +++
.../inlong/dataproxy/consts/SourceConstants.java | 108 ++++
.../inlong/dataproxy/metrics/audit/AuditUtils.java | 34 +-
.../dataproxy/sink/common/DefaultEventHandler.java | 10 +-
.../inlong/dataproxy/sink/common/EventHandler.java | 6 +-
.../dataproxy/sink/mq/MessageQueueZoneSink.java | 2 +-
.../sink/mq/MessageQueueZoneSinkContext.java | 9 +-
.../inlong/dataproxy/sink/mq/PackProfile.java | 2 +-
.../apache/inlong/dataproxy/source/BaseSource.java | 10 +-
.../dataproxy/source/ServerMessageFactory.java | 4 +-
.../dataproxy/source/ServerMessageHandler.java | 2 +-
.../inlong/dataproxy/source/SimpleHttpSource.java | 7 +-
.../inlong/dataproxy/source/SimpleTcpSource.java | 7 +-
.../inlong/dataproxy/source/SimpleUdpSource.java | 8 +-
.../inlong/dataproxy/source/SourceConstants.java | 197 -------
.../source/httpMsg/HttpMessageHandler.java | 8 +
.../dataproxy/source/v0msg/AbsV0MsgCodec.java | 2 +
.../inlong/dataproxy/source/v0msg/CodecBinMsg.java | 3 +
.../dataproxy/source/v0msg/CodecTextMsg.java | 3 +
.../inlong/dataproxy/utils/AddressUtils.java | 52 ++
.../config/holder/TestCommonConfigHolder.java | 2 +-
.../src/test/resources/common.properties | 10 +-
.../src/test/resources/metadata.json | 2 +-
inlong-dataproxy/pom.xml | 2 +-
inlong-distribution/pom.xml | 209 +++++--
.../src/main/assemblies/release.xml | 2 +-
.../src/main/assemblies/sort-connectors-v1.15.xml | 8 +
.../src/main/assemblies/sort-connectors-v1.18.xml | 49 ++
inlong-manager/manager-client-examples/pom.xml | 2 +-
inlong-manager/manager-client-tools/pom.xml | 2 +-
inlong-manager/manager-client/pom.xml | 2 +-
.../manager/client/api/InlongGroupContext.java | 2 +-
.../manager/client/api/impl/InlongGroupImpl.java | 2 +-
.../client/api/inner/client/InlongGroupClient.java | 25 +-
.../api/inner/client/InlongStreamClient.java | 11 +
.../client/api/inner/client/StreamSinkClient.java | 7 +
.../api/inner/client/StreamSourceClient.java | 11 +
.../client/api/inner/client/WorkflowClient.java | 21 +
.../manager/client/api/service/InlongGroupApi.java | 9 +-
.../client/api/service/InlongStreamApi.java | 4 +
.../manager/client/api/service/StreamSinkApi.java | 4 +
.../client/api/service/StreamSourceApi.java | 6 +
.../manager/client/api/service/WorkflowApi.java | 4 +
.../manager/client/api/util/ClientUtils.java | 2 +-
inlong-manager/manager-common/pom.xml | 2 +-
.../manager/common/consts/AuditSourceType.java | 25 -
.../manager/common/enums/AuditQuerySource.java | 9 +-
.../inlong/manager/common/enums/ErrorCodeEnum.java | 8 +
.../inlong/manager/common/enums/GroupStatus.java | 21 +-
.../inlong/manager/common/enums/ModuleType.java | 28 +
.../inlong/manager/common/enums/OperationType.java | 15 +-
.../inlong/manager/common/enums/ProcessName.java | 7 +-
.../manager/common/enums/SimpleGroupStatus.java | 4 +-
.../inlong/manager/common/enums/StreamStatus.java | 15 +-
inlong-manager/manager-dao/pom.xml | 2 +-
.../inlong/manager/dao/entity/AuditBaseEntity.java | 2 +-
.../manager/dao/entity/AuditSourceEntity.java | 46 --
.../manager/dao/entity/ClusterConfigEntity.java | 43 ++
.../manager/dao/entity/ModuleConfigEntity.java | 44 ++
.../manager/dao/entity/PackageConfigEntity.java | 44 ++
.../manager/dao/entity/SortConfigEntity.java | 47 ++
.../manager/dao/entity/StreamSourceEntity.java | 4 +-
.../manager/dao/mapper/AuditBaseEntityMapper.java | 3 +-
.../manager/dao/mapper/AuditEntityMapper.java | 1 +
.../dao/mapper/AuditSourceEntityMapper.java | 37 --
.../dao/mapper/ClusterConfigEntityMapper.java | 46 ++
.../manager/dao/mapper/ClusterSetMapper.java | 4 +
.../dao/mapper/InlongGroupEntityMapper.java | 5 +
.../dao/mapper/InlongGroupExtEntityMapper.java | 2 +
.../dao/mapper/ModuleConfigEntityMapper.java | 39 ++
.../dao/mapper/PackageConfigEntityMapper.java | 39 ++
.../manager/dao/mapper/SortConfigEntityMapper.java | 46 ++
.../manager/dao/mapper/StreamSinkEntityMapper.java | 2 +-
.../dao/mapper/StreamSourceEntityMapper.java | 15 +-
.../resources/mappers/AuditBaseEntityMapper.xml | 12 +-
.../resources/mappers/AuditSourceEntityMapper.xml | 71 ---
.../main/resources/mappers/ClusterConfigMapper.xml | 98 ++++
.../main/resources/mappers/ClusterSetMapper.xml | 6 +
.../resources/mappers/DataNodeEntityMapper.xml | 20 +-
.../resources/mappers/InlongGroupEntityMapper.xml | 24 +-
.../mappers/InlongGroupExtEntityMapper.xml | 7 +
.../main/resources/mappers/ModuleConfigMapper.xml | 104 ++++
.../main/resources/mappers/PackageConfigMapper.xml | 104 ++++
.../resources/mappers/SortConfigEntityMapper.xml | 118 ++++
.../resources/mappers/StreamSinkEntityMapper.xml | 12 +-
.../resources/mappers/StreamSourceEntityMapper.xml | 57 +-
inlong-manager/manager-docker/pom.xml | 2 +-
inlong-manager/manager-plugins/base/pom.xml | 49 +-
.../base/src/main/assembly/assembly.xml | 21 +
.../manager-plugins-flink-v1.13/pom.xml | 2 +-
.../manager-plugins-flink-v1.15/pom.xml | 2 +-
.../manager-plugins-flink-v1.18/pom.xml | 95 ++++
.../manager/plugin/flink/FlinkClientService.java | 0
inlong-manager/manager-plugins/pom.xml | 35 +-
inlong-manager/manager-pojo/pom.xml | 2 +-
.../manager/pojo/audit/AuditBaseResponse.java | 4 +-
.../inlong/manager/pojo/audit/AuditResponse.java | 51 ++
.../manager/pojo/audit/AuditSourceRequest.java | 68 ---
.../manager/pojo/audit/AuditSourceResponse.java | 73 ---
.../manager/pojo/cluster/ClusterNodeRequest.java | 3 +
.../manager/pojo/cluster/ClusterTagRequest.java | 3 +
.../manager/pojo/cluster/ClusterTagResponse.java | 3 +
.../pojo/cluster/InlongClusterTagExtParam.java | 114 ++++
.../pojo/cluster/agent/AgentClusterNodeDTO.java | 47 +-
.../cluster/agent/AgentClusterNodeRequest.java | 19 +
.../cluster/agent/AgentClusterNodeResponse.java | 19 +
.../pojo/cluster/kafka/KafkaClusterInfo.java | 6 +
.../pojo/cluster/kafka/KafkaClusterRequest.java | 6 +
.../inlong/manager/pojo/common/BatchResult.java | 39 ++
.../manager/pojo/consume/BriefMQMessage.java | 21 +
.../inlong/manager/pojo/group/GroupFullInfo.java | 41 ++
.../inlong/manager/pojo/group/InlongGroupInfo.java | 3 +
.../inlong/manager/pojo/module/ModuleDTO.java | 110 ++++
.../inlong/manager/pojo/module/ModuleHistory.java | 43 ++
.../manager/pojo/module/ModulePageRequest.java | 47 ++
.../inlong/manager/pojo/module/ModuleRequest.java | 72 +++
.../inlong/manager/pojo/module/ModuleResponse.java | 97 ++++
.../inlong/manager/pojo/module/PackageHistory.java | 42 ++
.../manager/pojo/module/PackagePageRequest.java | 47 ++
.../inlong/manager/pojo/module/PackageRequest.java | 54 ++
.../manager/pojo/module/PackageResponse.java | 75 +++
.../manager/pojo/node/cls/ClsDataNodeInfo.java | 2 +-
.../manager/pojo/node/kafka/KafkaDataNodeDTO.java | 78 +++
.../manager/pojo/node/kafka/KafkaDataNodeInfo.java | 61 ++
.../pojo/node/kafka/KafkaDataNodeRequest.java | 55 ++
.../pojo/node/pulsar/PulsarDataNodeDTO.java | 3 +
.../pojo/node/pulsar/PulsarDataNodeInfo.java | 3 +
.../pojo/node/pulsar/PulsarDataNodeRequest.java | 3 +
.../pojo/sort/node/base/LoadNodeProvider.java | 2 +-
.../pojo/sort/node/provider/PulsarProvider.java | 6 +-
.../pojo/sort/node/provider/StarRocksProvider.java | 2 +-
.../pojo/sort/node/provider/TubeMqProvider.java | 2 +-
.../sort/standalone/SortSourceClusterInfo.java | 2 +-
.../pojo/sort/standalone/SortSourceGroupInfo.java | 2 +-
.../sort/standalone/SortSourceStreamSinkInfo.java | 2 +-
.../manager/pojo/sort/util/FieldInfoUtils.java | 39 +-
.../manager/pojo/sort/util/FieldRelationUtils.java | 4 +-
.../inlong/manager/pojo/source/DataAddTaskDTO.java | 61 ++
.../manager/pojo/source/DataAddTaskRequest.java | 44 ++
.../inlong/manager/pojo/source/SourceRequest.java | 4 +-
.../inlong/manager/pojo/source/StreamSource.java | 8 +-
.../inlong/manager/pojo/source/SubSourceDTO.java | 61 --
.../pojo/source/file/FileDataAddTaskRequest.java | 43 ++
.../manager/pojo/source/mongodb/MongoDBSource.java | 3 +
.../pojo/source/mongodb/MongoDBSourceDTO.java | 3 +
.../pojo/source/mongodb/MongoDBSourceRequest.java | 3 +
.../manager/pojo/source/pulsar/PulsarSource.java | 9 +
.../pojo/source/pulsar/PulsarSourceDTO.java | 9 +
.../pojo/source/pulsar/PulsarSourceRequest.java | 9 +
.../manager/pojo/stream/InlongStreamBriefInfo.java | 3 -
.../manager/pojo/stream/InlongStreamExtParam.java | 3 -
.../manager/pojo/stream/InlongStreamInfo.java | 3 -
.../manager/pojo/stream/InlongStreamRequest.java | 3 -
.../manager/pojo/workflow/ProcessResponse.java | 2 +-
.../inlong/manager/pojo/workflow/TaskResponse.java | 2 +-
.../form/process/ApplyConsumeProcessForm.java | 8 +-
.../form/process/ApplyGroupProcessForm.java | 53 +-
.../workflow/form/process/BaseProcessForm.java | 7 +
.../form/process/ClusterResourceProcessForm.java | 56 ++
.../form/process/GroupResourceProcessForm.java | 7 +-
.../pojo/workflow/form/process/ProcessForm.java | 7 +-
.../workflow/form/task/InlongGroupApproveForm.java | 38 +-
.../manager/pojo/sort/util/FieldInfoUtilsTest.java | 12 +-
inlong-manager/manager-service/pom.xml | 14 +-
.../service/audit/AbstractAuditSourceOperator.java | 29 -
.../manager/service/audit/AuditRunnable.java | 185 ++++++
.../audit/AuditSourceClickhouseOperator.java | 48 --
.../audit/AuditSourceElasticsearchOperator.java | 50 --
.../service/audit/AuditSourceMysqlOperator.java | 49 --
.../service/audit/InlongAuditSourceOperator.java | 44 --
.../audit/InlongAuditSourceOperatorFactory.java | 47 --
.../cluster/InlongClusterProcessService.java | 102 ++++
.../service/cluster/InlongClusterServiceImpl.java | 40 +-
.../inlong/manager/service/core/AgentService.java | 5 +
.../inlong/manager/service/core/AuditService.java | 26 +-
.../manager/service/core/SortConfigLoader.java | 17 +
.../inlong/manager/service/core/SortService.java | 10 +
.../service/core/impl/AgentServiceImpl.java | 144 ++++-
.../service/core/impl/AuditServiceImpl.java | 451 ++-------------
.../service/core/impl/SortConfigLoaderImpl.java | 26 +
.../manager/service/core/impl/SortServiceImpl.java | 205 ++++++-
.../service/datatype/CsvDataTypeOperator.java | 78 +++
.../manager/service/datatype/DataTypeOperator.java | 54 ++
.../service/datatype/DataTypeOperatorFactory.java | 49 ++
.../service/group/InlongGroupProcessService.java | 37 +-
.../manager/service/group/InlongGroupService.java | 32 +-
.../service/group/InlongGroupServiceImpl.java | 75 +++
.../listener/ClusterTaskListenerFactory.java | 95 ++++
.../cluster/InitClusterCompleteListener.java | 53 ++
.../cluster/InitClusterFailedListener.java | 49 ++
.../listener/cluster/InitClusterListener.java | 49 ++
.../apply/ApproveConsumeProcessListener.java | 38 +-
.../group/UpdateGroupCompleteListener.java | 16 +-
.../group/apply/AfterApprovedTaskListener.java | 20 +-
.../group/apply/ApproveApplyProcessListener.java | 28 +-
.../group/apply/CancelApplyProcessListener.java | 33 +-
.../group/apply/RejectApplyProcessListener.java | 31 +-
.../listener/queue/ClusterConfigListener.java | 123 ++++
.../queue/ClusterQueueResourceListener.java | 81 +++
.../source/AbstractSourceOperateListener.java | 2 +-
.../listener/source/SourceRestartListener.java | 2 +-
.../listener/source/SourceStopListener.java | 2 +-
.../stream/UpdateStreamCompleteListener.java | 4 +-
.../listener/stream/UpdateStreamListener.java | 4 +-
.../service/message/DeserializeOperator.java | 6 +
.../message/InlongMsgDeserializeOperator.java | 42 +-
.../service/message/PbMsgDeserializeOperator.java | 51 +-
.../service/message/RawMsgDeserializeOperator.java | 32 +-
.../manager/service/module/ModuleService.java | 72 +++
.../manager/service/module/ModuleServiceImpl.java | 148 +++++
.../manager/service/module/PackageService.java | 72 +++
.../manager/service/module/PackageServiceImpl.java | 120 ++++
.../service/node/AbstractDataNodeOperator.java | 9 +-
.../manager/service/node/DataNodeOperator.java | 13 +-
.../service/node/cls/ClsDataNodeOperator.java | 10 +
.../node/es/ElasticsearchDataNodeOperator.java | 10 +
.../service/node/kafka/KafkaDataNodeOperator.java | 136 +++++
.../service/node/mysql/MySQLDataNodeOperator.java | 6 +-
.../node/pulsar/PulsarDataNodeOperator.java | 11 +
.../repository/DataProxyConfigRepository.java | 26 +-
.../repository/DataProxyConfigRepositoryV2.java | 48 +-
.../resource/queue/kafka/KafkaOperator.java | 10 +-
.../service/resource/queue/kafka/KafkaUtils.java | 2 +-
.../resource/queue/pulsar/PulsarOperator.java | 4 +
.../queue/pulsar/PulsarQueueResourceOperator.java | 44 +-
.../resource/queue/pulsar/QueryCountDownLatch.java | 55 ++
.../queue/pulsar/QueryLatestMessagesRunnable.java | 82 +++
.../resource/queue/tubemq/TubeMQOperator.java | 10 +-
.../AbstractStandaloneSinkResourceOperator.java | 12 +-
.../service/resource/sink/ck/ClickHouseConfig.java | 104 ----
.../resource/sink/es/ElasticsearchConfig.java | 5 -
.../resource/sort/DefaultSortConfigOperator.java | 363 +++++-------
.../resource/sort/SortFlinkConfigOperator.java | 309 ++++++++++
.../manager/service/sink/AbstractSinkOperator.java | 10 +
.../manager/service/sink/StreamSinkOperator.java | 9 +
.../manager/service/sink/StreamSinkService.java | 10 +
.../service/sink/StreamSinkServiceImpl.java | 25 +
.../manager/service/sink/cls/ClsSinkOperator.java | 22 +
.../service/sink/es/ElasticsearchSinkOperator.java | 22 +
.../service/sink/kafka/KafkaSinkOperator.java | 22 +
.../service/sink/pulsar/PulsarSinkOperator.java | 22 +
.../service/source/AbstractSourceOperator.java | 7 +
.../service/source/StreamSourceOperator.java | 10 +
.../service/source/StreamSourceService.java | 20 +
.../service/source/StreamSourceServiceImpl.java | 40 +-
.../service/source/file/FileSourceOperator.java | 43 +-
.../service/stream/InlongStreamProcessService.java | 8 +-
.../service/stream/InlongStreamService.java | 12 +
.../service/stream/InlongStreamServiceImpl.java | 62 ++
.../manager/service/task/DataCleansingTask.java | 8 +-
.../service/task/DeleteStreamSourceTask.java | 2 +-
.../service/workflow/WorkflowServiceImpl.java | 4 +-
.../SyncClusterResourceWorkflowDefinition.java | 96 ++++
.../service/cluster/InlongClusterServiceTest.java | 9 +-
.../service/core/impl/AgentServiceTest.java | 16 +-
.../service/core/impl/AuditServiceTest.java | 105 ----
.../group/InlongGroupProcessServiceTest.java | 2 +-
.../resource/queue/kafka/KafkaOperatorTest.java | 4 +
.../source/listener/StreamSourceListenerTest.java | 2 +-
inlong-manager/manager-test/pom.xml | 2 +-
.../main/resources/h2/apache_inlong_manager.sql | 103 +++-
inlong-manager/manager-web/pom.xml | 2 +-
.../manager-web/sql/apache_inlong_manager.sql | 108 +++-
inlong-manager/manager-web/sql/changes-1.12.0.sql | 84 +++
.../manager/web/controller/AuditController.java | 16 -
.../web/controller/InlongClusterController.java | 16 +
.../web/controller/InlongGroupController.java | 21 +
.../web/controller/InlongStreamController.java | 10 +
.../manager/web/controller/ModuleController.java | 84 +++
.../manager/web/controller/PackageController.java | 84 +++
.../web/controller/StreamSinkController.java | 8 +
.../web/controller/StreamSourceController.java | 18 +
.../controller/openapi/InstallerController.java | 49 ++
.../openapi/OpenInLongGroupController.java | 10 +
.../openapi/OpenInLongStreamController.java | 9 +
.../openapi/OpenStreamSinkController.java | 8 +
.../openapi/OpenStreamSourceController.java | 11 +
.../web/controller/openapi/SortController.java | 9 +
.../src/main/resources/application-dev.properties | 49 +-
.../src/main/resources/application-prod.properties | 48 +-
.../src/main/resources/application-test.properties | 48 +-
.../org/apache/inlong/manager/web/WebBaseTest.java | 2 +-
.../web/controller/HeartbeatControllerTest.java | 301 ++++++++++
inlong-manager/manager-workflow/pom.xml | 2 +-
.../event/task/ClusterOperateListener.java | 42 ++
inlong-manager/pom.xml | 2 +-
inlong-sdk/dataproxy-sdk/pom.xml | 2 +-
inlong-sdk/pom.xml | 3 +-
inlong-sdk/sdk-common/pom.xml | 2 +-
inlong-sdk/sort-sdk/pom.xml | 4 +-
inlong-sdk/transform-sdk/pom.xml | 107 ++++
.../inlong/sdk/transform/decode/CsvSourceData.java | 60 ++
.../sdk/transform/decode/CsvSourceDecoder.java | 88 +++
.../inlong/sdk/transform/decode/JsonNode.java | 51 ++
.../sdk/transform/decode/JsonSourceData.java | 134 +++++
.../sdk/transform/decode/JsonSourceDecoder.java | 123 ++++
.../inlong/sdk/transform/decode/KvSourceData.java | 59 ++
.../sdk/transform/decode/KvSourceDecoder.java | 74 +++
.../inlong/sdk/transform/decode/KvUtils.java | 579 +++++++++++++++++++
.../apache/inlong/sdk/transform/decode/PbNode.java | 95 ++++
.../inlong/sdk/transform/decode/PbSourceData.java | 240 ++++++++
.../sdk/transform/decode/PbSourceDecoder.java | 165 ++++++
.../inlong/sdk/transform/decode/SourceData.java | 30 +
.../inlong/sdk/transform/decode/SourceDecoder.java | 30 +
.../inlong/sdk/transform/decode/SplitUtils.java | 164 ++++++
.../sdk/transform/decode/TransformException.java | 39 ++
.../sdk/transform/encode/CsvSinkEncoder.java | 85 +++
.../sdk/transform/encode/DefaultSinkData.java | 60 ++
.../inlong/sdk/transform/encode/EscapeUtils.java | 83 +++
.../inlong/sdk/transform/encode/KvSinkEncoder.java | 72 +++
.../inlong/sdk/transform/encode/SinkData.java | 33 ++
.../inlong/sdk/transform/encode/SinkEncoder.java | 32 ++
.../inlong/sdk/transform/pojo/CsvSinkInfo.java | 99 ++++
.../inlong/sdk/transform/pojo/CsvSourceInfo.java | 99 ++++
.../inlong/sdk/transform/pojo/FieldInfo.java | 29 +
.../inlong/sdk/transform/pojo/JsonSourceInfo.java | 56 ++
.../inlong/sdk/transform/pojo/KvSinkInfo.java | 58 ++
.../inlong/sdk/transform/pojo/KvSourceInfo.java | 59 ++
.../inlong/sdk/transform/pojo/PbSourceInfo.java | 98 ++++
.../inlong/sdk/transform/pojo/ProtocolType.java | 61 ++
.../apache/inlong/sdk/transform/pojo/SinkInfo.java | 87 +++
.../inlong/sdk/transform/pojo/SourceInfo.java | 69 +++
.../inlong/sdk/transform/pojo/TransformConfig.java | 96 ++++
.../sdk/transform/process/TransformProcessor.java | 189 +++++++
.../transform/process/function/ConcatFunction.java | 70 +++
.../transform/process/function/NowFunction.java | 48 ++
.../transform/process/operator/AndOperator.java | 49 ++
.../process/operator/EqualsToOperator.java | 52 ++
.../process/operator/ExpressionOperator.java | 28 +
.../operator/GreaterThanEqualsOperator.java | 52 ++
.../process/operator/GreaterThanOperator.java | 52 ++
.../process/operator/MinorThanEqualsOperator.java | 52 ++
.../process/operator/MinorThanOperator.java | 52 ++
.../process/operator/NotEqualsToOperator.java | 52 ++
.../transform/process/operator/NotOperator.java | 47 ++
.../transform/process/operator/OperatorTools.java | 163 ++++++
.../sdk/transform/process/operator/OrOperator.java | 49 ++
.../process/operator/ParenthesisOperator.java | 47 ++
.../transform/process/parser/AdditionParser.java | 56 ++
.../sdk/transform/process/parser/ColumnParser.java | 52 ++
.../transform/process/parser/DivisionParser.java | 56 ++
.../sdk/transform/process/parser/LongParser.java | 46 ++
.../process/parser/MultiplicationParser.java | 56 ++
.../process/parser/ParenthesisParser.java | 47 ++
.../sdk/transform/process/parser/StringParser.java | 46 ++
.../process/parser/SubtractionParser.java | 56 ++
.../sdk/transform/process/parser/ValueParser.java | 29 +
.../sdk/transform/pojo/TestTransformConfig.java | 118 ++++
.../transform/process/TestTransformProcessor.java | 339 +++++++++++
inlong-sort-standalone/pom.xml | 2 +-
.../sort-standalone-common/pom.xml | 2 +-
.../loader/ClassResourceQueryConsumeConfig.java | 4 +
.../sort/standalone/config/pojo/type/DataType.java | 70 ---
.../sort-standalone-dist/pom.xml | 2 +-
.../sort-standalone-source/pom.xml | 2 +-
.../sort/standalone/metrics/audit/AuditUtils.java | 2 +-
.../sort/standalone/sink/kafka/KafkaIdConfig.java | 12 +-
.../sink/pulsar/PulsarFederationSinkContext.java | 6 +-
.../standalone/sink/pulsar/PulsarIdConfig.java | 12 +-
.../sink/pulsar/PulsarProducerCluster.java | 2 +-
inlong-sort/pom.xml | 12 +-
inlong-sort/sort-api/pom.xml | 2 +-
inlong-sort/sort-common/pom.xml | 2 +-
.../org/apache/inlong/sort/protocol/FieldInfo.java | 2 +-
.../apache/inlong/sort/protocol/node/LoadNode.java | 1 -
.../protocol/node/extract/MongoExtractNode.java | 2 +-
.../protocol/node/extract/PulsarExtractNode.java | 35 +-
.../sort/protocol/node/load/HiveLoadNode.java | 4 +-
.../apache/inlong/sort/protocol/FieldInfoTest.java | 2 +-
.../apache/inlong/sort/protocol/GroupInfoTest.java | 10 +-
.../inlong/sort/protocol/StreamInfoTest.java | 10 +-
.../node/extract/DorisExtractNodeTest.java | 8 +-
.../protocol/node/extract/HudiExtractNodeTest.java | 8 +-
.../node/extract/KafkaExtractNodeTest.java | 4 +-
.../node/extract/MongoExtractNodeTest.java | 4 +-
.../node/extract/MySqlExtractNodeTest.java | 2 +-
.../node/extract/OracleExtractNodeTest.java | 4 +-
.../node/extract/PostgresExtractNodeTest.java | 4 +-
.../node/extract/PulsarExtractNodeTest.java | 8 +-
.../node/extract/RedisExtractNodeTest.java | 4 +-
.../node/extract/SqlServerExtractNodeTest.java | 4 +-
.../node/extract/TubeMQExtractNodeTest.java | 8 +-
.../protocol/node/load/ClickHouseLoadNodeTest.java | 2 +-
.../sort/protocol/node/load/DorisLoadNodeTest.java | 8 +-
.../protocol/node/load/ElasticSearchNodeTest.java | 2 +-
.../protocol/node/load/GreenplumLoadNodeTest.java | 2 +-
.../sort/protocol/node/load/HbaseLoadNodeTest.java | 2 +-
.../sort/protocol/node/load/HiveLoadNodeTest.java | 4 +-
.../sort/protocol/node/load/HudiLoadNodeTest.java | 2 +-
.../protocol/node/load/IcebergLoadNodeTest.java | 2 +-
.../sort/protocol/node/load/KafkaLoadNodeTest.java | 2 +-
.../sort/protocol/node/load/MySqlLoadNodeTest.java | 2 +-
.../protocol/node/load/OracleLoadNodeTest.java | 2 +-
.../protocol/node/load/PostgresLoadNodeTest.java | 2 +-
.../protocol/node/load/SqlServerLoadNodeTest.java | 4 +-
.../node/load/TDSQLPostgresLoadNodeTest.java | 2 +-
.../protocol/node/transform/DistinctNodeTest.java | 4 +-
.../protocol/transformation/FieldRelationTest.java | 2 +-
.../transformation/WatermarkFieldTest.java | 2 +-
.../transformation/function/AddFunctionTest.java | 2 +-
.../function/BetweenFunctionTest.java | 2 +-
.../function/CascadeFunctionWrapperTest.java | 2 +-
.../function/HopEndFunctionTest.java | 2 +-
.../transformation/function/HopFunctionTest.java | 2 +-
.../function/HopStartFunctionTest.java | 2 +-
.../function/MultiValueFilterFunctionTest.java | 2 +-
.../function/RegexpReplaceFirstFunctionTest.java | 2 +-
.../function/RegexpReplaceFunctionTest.java | 2 +-
.../function/SessionEndFunctionTest.java | 2 +-
.../function/SessionFunctionTest.java | 2 +-
.../function/SessionStartFunctionTest.java | 2 +-
.../function/SingleValueFilterFunctionTest.java | 2 +-
.../function/SplitIndexFunctionTest.java | 2 +-
.../function/SubtractFunctionTest.java | 2 +-
.../function/TumbleEndFunctionTest.java | 2 +-
.../function/TumbleFunctionTest.java | 2 +-
.../function/TumbleStartFunctionTest.java | 2 +-
.../relation/FullOuterJoinNodeRelationTest.java | 2 +-
.../relation/InnerJoinNodeRelationTest.java | 2 +-
.../relation/InnerTemporalJoinRelationTest.java | 4 +-
.../relation/IntervalJoinRelationTest.java | 2 +-
.../relation/LeftOuterJoinNodeRelationTest.java | 2 +-
.../relation/LeftTemporalJoinRelationTest.java | 4 +-
.../relation/RightOuterJoinNodeRelationTest.java | 2 +-
inlong-sort/sort-core/pom.xml | 120 +++-
.../inlong/sort/parser/impl/FlinkSqlParser.java | 8 +-
.../sort/function/CascadeFunctionWrapperTest.java | 2 +-
.../inlong/sort/function/EncryptFunctionTest.java | 2 +-
.../sort/function/JsonGetterFunctionTest.java | 2 +-
.../function/RegexpReplaceFirstFunctionTest.java | 2 +-
.../sort/function/RegexpReplaceFunctionTest.java | 2 +-
.../sort/function/SplitIndexFunctionTest.java | 2 +-
.../inlong/sort/parser/AllMigrateMongoDBTest.java | 2 +-
.../inlong/sort/parser/AllMigrateOracleTest.java | 4 +-
.../sort/parser/AllMigratePostgreSQLTest.java | 2 +-
.../apache/inlong/sort/parser/AllMigrateTest.java | 4 +-
.../parser/AllMigrateWithSpecifyingFieldTest.java | 2 +-
.../sort/parser/ClickHouseSqlParserTest.java | 4 +-
.../sort/parser/CustomFunctionSqlParseTest.java | 6 +-
.../sort/parser/DataTypeConvertSqlParseTest.java | 6 +-
.../sort/parser/DecimalFormatSqlParseTest.java | 4 +-
.../sort/parser/DistinctNodeSqlParseTest.java | 10 +-
.../DorisExtractNodeToDorisLoadNodeTest.java | 8 +-
.../DorisExtractNodeToMySqlLoadNodeTest.java | 8 +-
.../inlong/sort/parser/DorisMultipleSinkTest.java | 2 +-
.../inlong/sort/parser/ESMultipleSinkTest.java | 2 +-
.../sort/parser/ElasticsearchSqlParseTest.java | 2 +-
.../sort/parser/FilesystemSqlParserTest.java | 6 +-
.../apache/inlong/sort/parser/FilterParseTest.java | 10 +-
.../inlong/sort/parser/FlinkSqlParserTest.java | 10 +-
.../sort/parser/FullOuterJoinSqlParseTest.java | 10 +-
.../sort/parser/GreenplumLoadSqlParseTest.java | 6 +-
.../sort/parser/HbaseLoadFlinkSqlParseTest.java | 4 +-
.../inlong/sort/parser/HudiNodeSqlParserTest.java | 10 +-
.../parser/Iceberg2StarRocksSqlParserTest.java | 4 +-
.../sort/parser/IcebergNodeSqlParserTest.java | 10 +-
.../sort/parser/InnerJoinRelationSqlParseTest.java | 10 +-
.../parser/IntervalJoinRelationSqlParseTest.java | 8 +-
.../inlong/sort/parser/KafkaLoadSqlParseTest.java | 8 +-
.../inlong/sort/parser/KafkaSqlParseTest.java | 6 +-
.../sort/parser/LeftOuterJoinSqlParseTest.java | 10 +-
.../inlong/sort/parser/MetaFieldSyncTest.java | 10 +-
.../sort/parser/MongoExtractFlinkSqlParseTest.java | 8 +-
.../MySqlExtractNodeToDorisLoadNodeTest.java | 4 +-
.../inlong/sort/parser/MySqlLoadSqlParseTest.java | 6 +-
.../MySqlTemporalJoinRelationSqlParseTest.java | 8 +-
.../sort/parser/OracleExtractSqlParseTest.java | 8 +-
.../inlong/sort/parser/OracleLoadSqlParseTest.java | 6 +-
.../parser/PostgresExtractFlinkSqlParseTest.java | 8 +-
.../parser/PostgresLoadNodeFlinkSqlParseTest.java | 4 +-
.../inlong/sort/parser/PulsarSqlParserTest.java | 8 +-
.../inlong/sort/parser/RedisNodeSqlParserTest.java | 10 +-
.../RedisTemporalJoinRelationSqlParseTest.java | 8 +-
.../sort/parser/RightOuterJoinSqlParseTest.java | 10 +-
.../sort/parser/SqlServerNodeSqlParseTest.java | 6 +-
.../TDSQLPostgresLoadNodeFlinkSqlParseTest.java | 4 +-
.../inlong/sort/parser/TubeMQNodeSqlParseTest.java | 8 +-
.../inlong/sort/parser/UnionSqlParseTest.java | 10 +-
inlong-sort/sort-dist/pom.xml | 47 +-
inlong-sort/sort-end-to-end-tests/pom.xml | 5 +-
.../sort-end-to-end-tests-v1.13/pom.xml | 4 +-
.../apache/inlong/sort/tests/ClickHouseITCase.java | 162 ------
.../apache/inlong/sort/tests/KafkaE2EITCase.java | 247 --------
.../inlong/sort/tests/Mysql2ClickHouseTest.java | 164 ++++++
.../apache/inlong/sort/tests/Mysql2KafkaTest.java | 249 ++++++++
.../sort/tests/utils/FlinkContainerTestEnv.java | 8 +-
.../inlong/sort/tests/utils/MySqlContainer.java | 7 +-
.../src/test/resources/docker/mysql/my.cnf | 3 +-
.../src/test/resources/docker/mysql/setup.sql | 1 +
.../test/resources/flinkSql/clickhouse_test.sql | 2 +-
.../src/test/resources/flinkSql/kafka_test.sql | 3 +-
.../src/test/resources/groupFile/kafka_test.json | 2 +-
.../src/test/resources/log4j2-test.properties | 2 +-
.../sort-end-to-end-tests-v1.15/pom.xml | 23 +-
.../inlong/sort/tests/Kafka2StarRocksTest.java | 227 ++++++++
.../apache/inlong/sort/tests/KafkaE2EITCase.java | 230 --------
.../inlong/sort/tests/Mongodb2StarRocksTest.java | 201 +++++++
.../sort/tests/MongodbToStarRocksITCase.java | 203 -------
.../inlong/sort/tests/Mysql2StarRocksTest.java | 166 ++++++
.../inlong/sort/tests/MysqlToRocksITCase.java | 169 ------
.../inlong/sort/tests/Postgres2StarRocksTest.java | 167 ++++++
.../sort/tests/PostgresToStarRocksITCase.java | 174 ------
.../apache/inlong/sort/tests/RedisToRedisTest.java | 141 +++++
.../inlong/sort/tests/Sqlserver2StarRocksTest.java | 193 +++++++
.../sort/tests/SqlserverToStarRocksITCase.java | 196 -------
.../sort/tests/utils/FlinkContainerTestEnv.java | 58 +-
.../tests/utils/FlinkContainerTestEnvJRE11.java | 55 ++
.../tests/utils/FlinkContainerTestEnvJRE8.java | 55 ++
.../inlong/sort/tests/utils/RedisContainer.java | 37 ++
.../inlong/sort/tests/utils/StarRocksManager.java | 3 +-
.../test/resources/docker/starrocks/start_fe_be.sh | 2 +-
.../src/test/resources/flinkSql/redis_test.sql | 208 +++++++
.../src/test/resources/log4j2-test.properties | 12 +-
inlong-sort/sort-flink/base/pom.xml | 107 +++-
.../org/apache/inlong/sort/base/Constants.java | 14 +
.../inlong/sort/base/metric/SinkMetricData.java | 2 +-
.../inlong/sort/base/metric/SourceMetricData.java | 2 +-
inlong-sort/sort-flink/cdc-base/pom.xml | 2 +-
inlong-sort/sort-flink/pom.xml | 11 +-
inlong-sort/sort-flink/sort-flink-v1.13/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/doris/pom.xml | 2 +-
.../sort-connectors/elasticsearch-6/pom.xml | 2 +-
.../sort-connectors/elasticsearch-7/pom.xml | 2 +-
.../sort-connectors/elasticsearch-base/pom.xml | 2 +-
.../sort-connectors/filesystem/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/hbase/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/hive/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/hudi/pom.xml | 2 +-
.../sort-connectors/iceberg/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/jdbc/pom.xml | 2 +-
.../jdbc/internal/JdbcBatchingOutputFormat.java | 5 +-
.../internal/JdbcMultiBatchingOutputFormat.java | 4 +
.../sort-flink-v1.13/sort-connectors/kafka/pom.xml | 5 +-
.../sort-flink-v1.13/sort-connectors/kudu/pom.xml | 2 +-
.../apache/inlong/sort/kudu/common/KuduUtils.java | 24 +-
.../inlong/sort/connector/kudu/KuduTestBase.java | 10 +-
.../sort-connectors/mongodb-cdc/pom.xml | 2 +-
.../sort-connectors/mysql-cdc/pom.xml | 2 +-
.../sort-connectors/oracle-cdc/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/pom.xml | 2 +-
.../sort-connectors/postgres-cdc/pom.xml | 2 +-
.../sort-connectors/pulsar/pom.xml | 2 +-
.../pulsar/table/PulsarDynamicTableFactory.java | 4 +
.../sort-flink-v1.13/sort-connectors/redis/pom.xml | 2 +-
.../sort-connectors/sqlserver-cdc/pom.xml | 2 +-
.../sort-connectors/starrocks/pom.xml | 2 +-
.../sort-connectors/tubemq/pom.xml | 2 +-
.../sort-flink-dependencies/pom.xml | 2 +-
inlong-sort/sort-flink/sort-flink-v1.15/pom.xml | 3 +-
.../sort-flink-v1.15/sort-connectors/hbase/pom.xml | 2 +-
.../sort-flink-v1.15/sort-connectors/hudi/pom.xml | 2 +-
.../sort-connectors/iceberg/pom.xml | 2 +-
.../sort-flink-v1.15/sort-connectors/kafka/pom.xml | 2 +-
.../sort-connectors/mongodb-cdc/pom.xml | 2 +-
.../sort-connectors/mysql-cdc/pom.xml | 2 +-
.../sort-flink-v1.15/sort-connectors/pom.xml | 3 +-
.../sort-connectors/postgres-cdc/pom.xml | 2 +-
.../sort-connectors/pulsar/pom.xml | 2 +-
.../sort/pulsar/table/PulsarTableSource.java | 5 +
.../sort-flink-v1.15/sort-connectors/redis/pom.xml | 199 +++++++
.../sort/redis/common/config/RedisDataType.java | 0
.../redis/common/config/RedisLookupOptions.java | 117 ++++
.../sort/redis/common/config/RedisOptions.java | 0
.../redis/common/config/SchemaMappingMode.java | 0
.../handler/FlinkJedisClusterConfigHandler.java | 0
.../handler/FlinkJedisSentinelConfigHandler.java | 0
.../handler/FlinkJedisStandaloneConfigHandler.java | 0
.../container/InlongRedisClusterContainer.java | 0
.../container/InlongRedisCommandsContainer.java | 0
.../common/container/InlongRedisContainer.java | 0
.../container/RedisCommandsContainerBuilder.java | 0
.../common/descriptor/InlongRedisValidator.java | 0
.../common/handler/InlongJedisConfigHandler.java | 37 ++
.../redis/common/handler/RedisMapperHandler.java | 0
.../sort/redis/common/mapper/RedisCommand.java | 0
.../common/mapper/RedisCommandDescription.java | 0
.../sort/redis/common/mapper/RedisMapper.java | 0
.../sort/redis/common/mapper/row/GetMapper.java | 0
.../sort/redis/common/mapper/row/HgetMapper.java | 0
.../redis/common/mapper/row/RowRedisMapper.java | 0
.../redis/common/mapper/row/ZrevrankMapper.java | 0
.../sort/redis/common/mapper/row/ZscoreMapper.java | 0
.../sort/redis/common/schema/RedisSchema.java | 0
.../redis/common/schema/RedisSchemaFactory.java | 0
.../sort/redis/common/schema/StateEncoder.java | 0
.../common/schema/impl/AbstractRedisSchema.java | 109 ++++
.../schema/impl/BitmapStaticKvPairSchema.java | 80 +++
.../common/schema/impl/HashDynamicSchema.java | 0
.../common/schema/impl/HashStaticKvPairSchema.java | 74 +++
.../schema/impl/HashStaticPrefixMatchSchema.java | 0
.../common/schema/impl/PlainPrefixMatchSchema.java | 0
.../sort/redis/sink/AbstractRedisSinkFunction.java | 334 +++++++++++
.../sort/redis/sink/RedisBitmapSinkFunction.java | 0
.../sort/redis/sink/RedisDynamicTableSink.java | 177 ++++++
.../sort/redis/sink/RedisHashSinkFunction.java | 0
.../sort/redis/sink/RedisPlainSinkFunction.java | 0
.../sort/redis/source/RedisDynamicTableSource.java | 93 +++
.../redis/source/RedisRowDataLookupFunction.java | 154 +++++
.../sort/redis/table/RedisDynamicTableFactory.java | 212 +++++++
.../inlong/sort/redis/table/SchemaValidator.java | 0
...ng.connectors.redis.common.hanlder.RedisHandler | 0
.../org.apache.flink.table.factories.Factory | 0
.../sort/redis/RedisDynamicTableFactoryTest.java | 176 ++++++
.../apache/inlong/sort/redis/RedisTableTest.java | 627 +++++++++++++++++++++
.../sort-connectors/sqlserver-cdc/pom.xml | 2 +-
.../sort-connectors/starrocks/pom.xml | 2 +-
.../sort-connectors/tubemq/pom.xml | 2 +-
.../sort-flink-dependencies/pom.xml | 2 +-
inlong-sort/sort-flink/sort-flink-v1.18/pom.xml | 44 ++
.../sort-flink-v1.18/sort-connectors/jdbc/pom.xml | 117 ++++
.../jdbc/internal/GenericJdbcSinkFunction.java | 82 +++
.../sort/jdbc/internal/JdbcOutputFormat.java | 499 ++++++++++++++++
.../jdbc/internal/TableJdbcUpsertOutputFormat.java | 225 ++++++++
.../sort/jdbc/table/JdbcDynamicTableFactory.java | 357 ++++++++++++
.../sort/jdbc/table/JdbcDynamicTableSink.java | 148 +++++
.../sort/jdbc/table/JdbcOutputFormatBuilder.java | 283 ++++++++++
.../org.apache.flink.table.factories.Factory | 0
.../sort-flink-v1.18/sort-connectors/pom.xml | 50 ++
.../sort-connectors/pulsar/pom.xml | 121 ++++
.../sort/pulsar/table/PulsarTableFactory.java | 312 ++++++++++
.../sort/pulsar/table/PulsarTableOptionUtils.java | 322 +++++++++++
.../sort/pulsar/table/PulsarTableOptions.java | 295 ++++++++++
.../pulsar/table/PulsarTableValidationUtils.java | 197 +++++++
.../source/PulsarTableDeserializationSchema.java | 132 +++++
.../PulsarTableDeserializationSchemaFactory.java | 241 ++++++++
.../pulsar/table/source/PulsarTableSource.java | 227 ++++++++
.../org.apache.flink.table.factories.Factory | 16 +
.../sort-flink-dependencies/pom.xml | 80 +++
inlong-sort/sort-formats/format-common/pom.xml | 2 +-
.../inlong/sort/formats/base/TableFormatUtils.java | 184 +++---
.../sort/formats/base/TextFormatBuilder.java | 2 +-
.../sort/formats/base/TextFormatDescriptor.java | 6 +-
.../sort/formats/common/ArrayFormatInfo.java | 83 ---
.../inlong/sort/formats/common/ArrayTypeInfo.java | 77 ---
.../sort/formats/common/BasicFormatInfo.java | 42 --
.../sort/formats/common/BinaryFormatInfo.java | 96 ----
.../inlong/sort/formats/common/BinaryTypeInfo.java | 45 --
.../sort/formats/common/BooleanFormatInfo.java | 62 --
.../sort/formats/common/BooleanTypeInfo.java | 47 --
.../inlong/sort/formats/common/ByteFormatInfo.java | 62 --
.../inlong/sort/formats/common/ByteTypeInfo.java | 47 --
.../inlong/sort/formats/common/Constants.java | 26 -
.../inlong/sort/formats/common/DateFormatInfo.java | 162 ------
.../inlong/sort/formats/common/DateTypeInfo.java | 47 --
.../sort/formats/common/DecimalFormatInfo.java | 106 ----
.../sort/formats/common/DecimalTypeInfo.java | 47 --
.../sort/formats/common/DoubleFormatInfo.java | 62 --
.../inlong/sort/formats/common/DoubleTypeInfo.java | 47 --
.../sort/formats/common/FloatFormatInfo.java | 62 --
.../inlong/sort/formats/common/FloatTypeInfo.java | 47 --
.../inlong/sort/formats/common/FormatInfo.java | 61 --
.../inlong/sort/formats/common/FormatUtils.java | 75 ---
.../inlong/sort/formats/common/IntFormatInfo.java | 62 --
.../inlong/sort/formats/common/IntTypeInfo.java | 47 --
.../common/LocalZonedTimestampFormatInfo.java | 86 ---
.../common/LocalZonedTimestampTypeInfo.java | 45 --
.../inlong/sort/formats/common/LongFormatInfo.java | 62 --
.../inlong/sort/formats/common/LongTypeInfo.java | 47 --
.../inlong/sort/formats/common/MapFormatInfo.java | 101 ----
.../inlong/sort/formats/common/MapTypeInfo.java | 93 ---
.../inlong/sort/formats/common/NullFormatInfo.java | 50 --
.../inlong/sort/formats/common/NullTypeInfo.java | 45 --
.../inlong/sort/formats/common/RowFormatInfo.java | 123 ----
.../inlong/sort/formats/common/RowTypeInfo.java | 146 -----
.../sort/formats/common/ShortFormatInfo.java | 62 --
.../inlong/sort/formats/common/ShortTypeInfo.java | 47 --
.../sort/formats/common/StringFormatInfo.java | 62 --
.../inlong/sort/formats/common/StringTypeInfo.java | 47 --
.../inlong/sort/formats/common/TimeFormatInfo.java | 177 ------
.../inlong/sort/formats/common/TimeTypeInfo.java | 47 --
.../sort/formats/common/TimestampFormatInfo.java | 185 ------
.../sort/formats/common/TimestampTypeInfo.java | 47 --
.../inlong/sort/formats/common/TypeInfo.java | 51 --
.../sort/formats/common/VarBinaryFormatInfo.java | 111 ----
.../sort/formats/common/VarBinaryTypeInfo.java | 48 --
.../sort/formats/common/VarCharFormatInfo.java | 84 ---
.../sort/formats/common/ArrayFormatInfoTest.java | 7 +
.../sort/formats/common/BinaryFormatInfoTest.java | 3 +
.../sort/formats/common/BooleanFormatInfoTest.java | 3 +
.../sort/formats/common/ByteFormatInfoTest.java | 3 +
.../sort/formats/common/DateFormatInfoTest.java | 3 +
.../sort/formats/common/DecimalFormatInfoTest.java | 3 +
.../sort/formats/common/DoubleFormatInfoTest.java | 3 +
.../sort/formats/common/FloatFormatInfoTest.java | 3 +
.../sort/formats/common/FormatInfoTestBase.java | 3 +
.../sort/formats/common/IntFormatInfoTest.java | 3 +
.../common/LocalZonedTimestampFormatInfoTest.java | 3 +
.../sort/formats/common/LongFormatInfoTest.java | 3 +
.../sort/formats/common/MapFormatInfoTest.java | 9 +
.../sort/formats/common/RowFormatInfoTest.java | 7 +
.../sort/formats/common/ShortFormatInfoTest.java | 3 +
.../sort/formats/common/StringFormatInfoTest.java | 3 +
.../sort/formats/common/TimeFormatInfoTest.java | 3 +
.../formats/common/TimestampFormatInfoTest.java | 3 +
.../formats/common/VarBinaryFormatInfoTest.java | 3 +
.../sort/formats/common/VarCharFormatInfoTest.java | 3 +
.../sort-formats/format-row/format-base/pom.xml | 2 +-
.../sort/formats/base/TableFormatUtilsTest.java | 4 +-
.../sort-formats/format-row/format-csv/pom.xml | 2 +-
.../sort/formats/csv/CsvDeserializationSchema.java | 4 +-
.../inlong/sort/formats/csv/CsvFormatBuilder.java | 2 +-
.../inlong/sort/formats/csv/CsvFormatFactory.java | 6 +-
.../sort/formats/csv/CsvSerializationSchema.java | 4 +-
.../formats/csv/CsvDeserializationSchemaTest.java | 30 +-
.../sort/formats/csv/CsvFormatFactoryTest.java | 12 +-
.../formats/csv/CsvSerializationSchemaTest.java | 30 +-
.../format-row/format-inlongmsg-base/pom.xml | 2 +-
.../InLongMsgMixedFormatConverterBuilder.java | 2 +-
.../sort/formats/inlongmsg/InLongMsgUtils.java | 6 +-
.../format-row/format-inlongmsg-binlog/pom.xml | 2 +-
.../formats/inlongmsgbinlog/InLongMsgBinlog.java | 6 +-
.../InLongMsgBinlogFormatBuilder.java | 2 +-
.../InLongMsgBinlogFormatDeserializer.java | 2 +-
.../InLongMsgBinlogFormatFactory.java | 2 +-
.../InLongMsgBinlogMixedFormatConverter.java | 2 +-
.../inlongmsgbinlog/InLongMsgBinlogUtils.java | 11 +-
.../InLongMsgBinlogFormatDeserializerTest.java | 8 +-
.../InLongMsgBinlogFormatFactoryTest.java | 10 +-
.../inlongmsgbinlog/InLongMsgBinlogTest.java | 12 +-
.../format-row/format-inlongmsg-csv/pom.xml | 2 +-
.../InLongMsgCsvFormatDeserializer.java | 2 +-
.../inlongmsgcsv/InLongMsgCsvFormatFactory.java | 2 +-
.../InLongMsgCsvMixedFormatConverter.java | 2 +-
.../formats/inlongmsgcsv/InLongMsgCsvUtils.java | 8 +-
.../InLongMsgCsvFormatDeserializerTest.java | 8 +-
.../InLongMsgCsvFormatFactoryTest.java | 12 +-
.../format-row/format-inlongmsg-kv/pom.xml | 2 +-
.../inlongmsgkv/InLongMsgKvFormatDeserializer.java | 2 +-
.../inlongmsgkv/InLongMsgKvFormatFactory.java | 2 +-
.../InLongMsgKvMixedFormatConverter.java | 2 +-
.../sort/formats/inlongmsgkv/InLongMsgKvUtils.java | 7 +-
.../InLongMsgKvFormatDeserializerTest.java | 8 +-
.../inlongmsgkv/InLongMsgKvFormatFactoryTest.java | 10 +-
.../format-row/format-inlongmsg-pb/pom.xml | 2 +-
.../format-row/format-inlongmsg-tlogcsv/pom.xml | 2 +-
.../InLongMsgTlogCsvFormatDeserializer.java | 2 +-
.../InLongMsgTlogCsvFormatFactory.java | 2 +-
.../InLongMsgTlogCsvMixedFormatConverter.java | 2 +-
.../inlongmsgtlogcsv/InLongMsgTlogCsvUtils.java | 4 +-
.../InLongMsgTlogCsvFormatDeserializerTest.java | 8 +-
.../InLongMsgTlogCsvFormatFactoryTest.java | 12 +-
.../format-row/format-inlongmsg-tlogkv/pom.xml | 2 +-
.../InLongMsgTlogKvFormatDeserializer.java | 2 +-
.../InLongMsgTlogKvFormatFactory.java | 2 +-
.../InLongMsgTlogKvMixedFormatConverter.java | 2 +-
.../inlongmsgtlogkv/InLongMsgTlogKvUtils.java | 4 +-
.../InLongMsgTlogKvFormatDeserializerTest.java | 8 +-
.../InLongMsgTlogKvFormatFactoryTest.java | 12 +-
.../format-row/format-json-v1.13/pom.xml | 2 +-
.../format-row/format-json-v1.15/pom.xml | 2 +-
.../format-row/format-json-v1.18/pom.xml | 101 ++++
.../inlong/sort/formats/json/MysqlBinLogData.java | 48 ++
.../inlong/sort/formats/json/canal/CanalJson.java | 0
.../json/canal/CanalJsonDecodingFormat.java | 0
.../json/canal/CanalJsonDeserializationSchema.java | 0
.../canal/CanalJsonEnhancedDecodingFormat.java | 0
.../CanalJsonEnhancedDeserializationSchema.java | 0
.../canal/CanalJsonEnhancedEncodingFormat.java | 340 +++++++++++
.../json/canal/CanalJsonEnhancedFormatFactory.java | 0
.../CanalJsonEnhancedSerializationSchema.java | 0
.../json/canal/CanalJsonSerializationSchema.java | 0
.../inlong/sort/formats/json/canal/CanalUtils.java | 0
.../sort/formats/json/debezium/DebeziumJson.java | 0
.../json/debezium/DebeziumJsonDecodingFormat.java | 0
.../DebeziumJsonDeserializationSchema.java | 0
.../sort/formats/json/debezium/DebeziumUtils.java | 0
.../sort/formats/json/utils/FormatJsonUtil.java | 162 ++++++
.../org.apache.flink.table.factories.Factory | 0
.../canal/CanalJsonEnhancedFormatFactoryTest.java | 0
.../canal/CanalJsonEnhancedSerDeSchemaTest.java | 0
.../json/canal/CanalJsonSerializationTest.java | 0
.../json/canal/DebeziumJsonSerializationTest.java | 0
.../src/test/resources/canal-json-inlong-data.txt | 0
.../src/test/resources/log4j2-test.properties | 0
.../sort-formats/format-row/format-kv/pom.xml | 2 +-
.../sort/formats/kv/KvDeserializationSchema.java | 4 +-
.../inlong/sort/formats/kv/KvFormatBuilder.java | 2 +-
.../inlong/sort/formats/kv/KvFormatFactory.java | 6 +-
.../sort/formats/kv/KvSerializationSchema.java | 4 +-
.../formats/kv/KvDeserializationSchemaTest.java | 30 +-
.../sort/formats/kv/KvFormatFactoryTest.java | 14 +-
.../sort/formats/kv/KvSerializationSchemaTest.java | 30 +-
inlong-sort/sort-formats/format-row/pom.xml | 8 +-
.../format-inlongmsg-rowdata-base/pom.xml | 2 +-
.../InLongMsgMixedFormatConverterBuilder.java | 2 +-
.../sort/formats/inlongmsg/InLongMsgUtils.java | 4 +-
.../format-inlongmsg-rowdata-binlog/pom.xml | 2 +-
.../InLongMsgBinlogFormatDeserializer.java | 2 +-
...nLongMsgBinlogRowDataDeserializationSchema.java | 2 +-
.../inlongmsgbinlog/InLongMsgBinlogUtils.java | 20 +-
.../InLongMsgBinlogFormatFactoryTest.java | 12 +-
.../format-inlongmsg-rowdata-csv/pom.xml | 2 +-
.../InLongMsgCsvFormatDeserializer.java | 2 +-
.../InLongMsgCsvRowDataDeserializationSchema.java | 2 +-
.../formats/inlongmsgcsv/InLongMsgCsvUtils.java | 22 +-
.../InLongMsgCsvFormatDeserializerTest.java | 12 +-
.../InLongMsgCsvFormatFactoryTest.java | 16 +-
.../format-inlongmsg-rowdata-kv/pom.xml | 2 +-
.../inlongmsgkv/InLongMsgKvFormatDeserializer.java | 2 +-
.../InLongMsgKvRowDataDeserializationSchema.java | 2 +-
.../sort/formats/inlongmsgkv/InLongMsgKvUtils.java | 21 +-
.../InLongMsgKvFormatDeserializerTest.java | 8 +-
.../inlongmsgkv/InLongMsgKvFormatFactoryTest.java | 16 +-
.../format-inlongmsg-rowdata-pb/pom.xml | 2 +-
.../format-inlongmsg-rowdata-tlogcsv/pom.xml | 2 +-
.../InLongMsgTlogCsvDeserializationSchema.java | 2 +-
.../InLongMsgTlogCsvFormatDeserializer.java | 2 +-
.../inlongmsgtlogcsv/InLongMsgTlogCsvUtils.java | 4 +-
.../InLongMsgTlogCsvFormatDeserializerTest.java | 8 +-
.../InLongMsgTlogCsvFormatFactoryTest.java | 16 +-
.../format-inlongmsg-rowdata-tlogkv/pom.xml | 4 +-
.../InLongMsgTlogKvFormatDeserializer.java | 2 +-
.../inlongmsgtlogkv/InLongMsgTlogKvUtils.java | 4 +-
.../InLongMsgTlogKvFormatDeserializerTest.java | 8 +-
.../format-rowdata/format-rowdata-base/pom.xml | 2 +-
.../formats/base/TableFormatForRowDataUtils.java | 4 +-
.../sort/formats/base/TableFormatUtilsTest.java | 20 +-
.../format-rowdata/format-rowdata-csv/pom.xml | 2 +-
.../inlong/sort/formats/csv/CsvFormatFactory.java | 2 +-
.../csv/CsvRowDataDeserializationSchema.java | 4 +-
.../formats/csv/CsvRowDataSerializationSchema.java | 4 +-
.../sort/formats/csv/CsvFormatFactoryTest.java | 12 +-
.../csv/CsvRowDataDeserializationSchemaTest.java | 30 +-
.../csv/CsvRowDataSerializationSchemaTest.java | 30 +-
.../format-rowdata/format-rowdata-json/pom.xml | 2 +-
.../formats/json/JsonRowDeserializationSchema.java | 12 +-
.../formats/json/JsonRowSerializationSchema.java | 26 +-
.../json/JsonRowDeserializationSchemaTest.java | 32 +-
.../json/JsonRowSerializationSchemaTest.java | 32 +-
.../format-rowdata/format-rowdata-kv/pom.xml | 2 +-
.../inlong/sort/formats/kv/KvFormatBuilder.java | 2 +-
.../inlong/sort/formats/kv/KvFormatFactory.java | 2 +-
.../formats/kv/KvRowDataDeserializationSchema.java | 4 +-
.../formats/kv/KvRowDataSerializationSchema.java | 4 +-
.../sort/formats/kv/KvFormatFactoryTest.java | 12 +-
.../kv/KvRowDataDeserializationSchemaTest.java | 30 +-
.../kv/KvRowDataSerializationSchemaTest.java | 30 +-
inlong-sort/sort-formats/format-rowdata/pom.xml | 2 +-
inlong-sort/sort-formats/pom.xml | 107 +++-
inlong-tubemq/pom.xml | 2 +-
.../tubemq-client-go/config/config.go | 47 ++
.../tubemq-client-go/log/config.go | 43 ++
inlong-tubemq/tubemq-client/pom.xml | 2 +-
inlong-tubemq/tubemq-connectors/pom.xml | 2 +-
.../tubemq-connector-flink/pom.xml | 2 +-
.../tubemq-connector-flume/pom.xml | 2 +-
.../tubemq-connector-spark/pom.xml | 2 +-
inlong-tubemq/tubemq-core/pom.xml | 2 +-
inlong-tubemq/tubemq-docker/pom.xml | 2 +-
inlong-tubemq/tubemq-docker/tubemq-all/pom.xml | 2 +-
inlong-tubemq/tubemq-docker/tubemq-build/pom.xml | 2 +-
inlong-tubemq/tubemq-docker/tubemq-cpp/pom.xml | 2 +-
inlong-tubemq/tubemq-docker/tubemq-manager/pom.xml | 2 +-
inlong-tubemq/tubemq-example/pom.xml | 2 +-
inlong-tubemq/tubemq-manager/pom.xml | 2 +-
inlong-tubemq/tubemq-server/pom.xml | 2 +-
.../server/broker/stats/audit/AuditUtils.java | 2 +-
licenses/NOTICE | 2 +-
licenses/inlong-agent/NOTICE | 2 +-
licenses/inlong-audit/NOTICE | 2 +-
licenses/inlong-dashboard/NOTICE | 2 +-
licenses/inlong-dataproxy/NOTICE | 2 +-
licenses/inlong-manager/NOTICE | 2 +-
licenses/inlong-sort-connectors/LICENSE | 29 +
licenses/inlong-sort-connectors/NOTICE | 2 +-
licenses/inlong-sort-standalone/NOTICE | 2 +-
licenses/inlong-sort/NOTICE | 2 +-
licenses/inlong-tubemq-manager/NOTICE | 2 +-
licenses/inlong-tubemq-server/NOTICE | 2 +-
pom.xml | 4 +-
1202 files changed, 39401 insertions(+), 14930 deletions(-)
create mode 100644 .github/workflows/ci_ut_flink13.yml
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/KubernetesConstants.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/MetadataConstants.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/JobProfileDb.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/BinlogJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/BinlogTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/KafkaJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/KafkaTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/MongoJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/MongoTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/MqttJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/MqttTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/OracleJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/OracleTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/PostgreSQLJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/PostgreSQLTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/RedisJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/RedisTask.java
delete mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/SqlServerJob.java
create mode 100644 inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/SqlServerTask.java
create mode 100755 inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java
create mode 100644 inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java
create mode 100755 inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java
create mode 100644 inlong-agent/agent-installer/src/test/resources/conf/modules.json
create mode 100644 inlong-agent/agent-installer/src/test/resources/log4j2.xml
create mode 100644 inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/instance/CommonInstance.java
create mode 100644 inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/instance/MongoDBInstance.java
delete mode 100644 inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/KubernetesMetadataProvider.java
create mode 100644 inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/AbstractTask.java
create mode 100644 inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/MongoDBTask.java
delete mode 100644 inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/MetaDataUtils.java
delete mode 100644 inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/utils/MetaDataUtilsTest.java
create mode 100755 inlong-agent/bin/agent-config.sh
create mode 100644 inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditDimensions.java
create mode 100644 inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditValues.java
create mode 100644 inlong-audit/audit-service/pom.xml
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/AbstractCache.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/DayCache.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/HalfHourCache.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/HourCache.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/RealTimeQuery.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/TenMinutesCache.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/channel/DataQueue.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/ConfigConstants.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/Configuration.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/OpenApiConstants.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/SqlConstants.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/ApiType.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/AuditCycle.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/JdbcConfig.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/Request.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/SinkConfig.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/SourceConfig.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/StartEndTime.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/StatData.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/main/Application.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/SelectorChangeListenerImpl.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/api/Selector.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/api/SelectorChangeListener.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/api/SelectorConfig.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/api/SelectorFactory.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/impl/DBDataSource.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/impl/SelectorImpl.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/selector/task/DBMonitorTask.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/ApiService.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/ConfigService.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/EtlService.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/sink/CacheSink.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/sink/JdbcSink.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/source/JdbcSource.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/utils/CacheUtils.java
create mode 100644 inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/utils/JdbcUtils.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/config/ClickHouseConfig.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/config/DataServerConfig.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/config/ElasticsearchConfig.java
create mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/config/JdbcConfig.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/DruidConfig.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/DruidDataSourceProperties.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/dao/AuditDataDao.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/entities/AuditDataPo.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/entities/ClickHouseDataPo.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/entities/ESDataPo.java
create mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/db/entities/JdbcDataPo.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/ClickHouseService.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/ElasticsearchService.java
create mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/JdbcService.java
delete mode 100644 inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/MySqlService.java
delete mode 100644 inlong-audit/audit-store/src/main/resources/mapper/AuditDataDao.xml
delete mode 100644 inlong-audit/audit-store/src/main/resources/sql/apache_inlong_audit.sql
delete mode 100644 inlong-audit/audit-store/src/main/resources/sql/apache_inlong_audit_clickhouse.sql
delete mode 100644 inlong-audit/audit-store/src/test/java/org/apache/inlong/audit/service/ElasticsearchServiceTest.java
delete mode 100644 inlong-audit/audit-store/src/test/resources/mapper/AuditDataDao.xml
create mode 100644 inlong-audit/bin/service-start.sh
create mode 100644 inlong-audit/bin/service-stop.sh
create mode 100644 inlong-audit/conf/audit-service.properties
delete mode 100644 inlong-audit/conf/mapper/AuditDataDao.xml
delete mode 100644 inlong-audit/sql/apache_inlong_audit.sql
delete mode 100644 inlong-audit/sql/apache_inlong_audit_clickhouse.sql
create mode 100644 inlong-audit/sql/apache_inlong_audit_mysql.sql
create mode 100644 inlong-audit/sql/apache_inlong_audit_starrocks.sql
delete mode 100644 inlong-audit/sql/clickhouse-changes-1.9.0.sql
delete mode 100644 inlong-audit/sql/mysql-changes-1.9.0.sql
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/constant/CompressionType.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/constant/DataNodeType.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/constant/DeserializationType.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/constant/SinkType.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/enums/IndicatorType.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/ConfigRequest.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/installer/InstallerCode.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortClusterConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortTaskConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/DataFlowConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/SourceConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/CsvConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/DataTypeConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/KvConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/deserialization/DeserializationConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/deserialization/InlongMsgDeserializationConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/deserialization/InlongMsgPbDeserialiationConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/FieldConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ArrayFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ArrayTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/BasicFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/BinaryFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/BinaryTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/BooleanFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/BooleanTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ByteFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ByteTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/Constants.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/DateFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/DateTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/DecimalFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/DecimalTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/DoubleFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/DoubleTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/FloatFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/FloatTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/FormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/FormatUtils.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/IntFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/IntTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/LocalZonedTimestampFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/LocalZonedTimestampTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/LongFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/LongTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/MapFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/MapTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/NullFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/NullTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/RowFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/RowTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ShortFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ShortTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/StringFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/StringTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/TimeFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/TimeTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/TimestampFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/TimestampTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/TypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/VarBinaryFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/VarBinaryTypeInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/VarCharFormatInfo.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/sink/ClsSinkConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/sink/EsSinkConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/sink/PulsarSinkConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/sink/SinkConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/mq/MqClusterConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/mq/PulsarClusterConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/mq/TubeClusterConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/node/ClsNodeConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/node/EsNodeConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/node/NodeConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/node/PulsarNodeConfig.java
create mode 100644 inlong-common/src/main/java/org/apache/inlong/common/pojo/sortstandalone/SortConfigResponse.java
create mode 100644 inlong-dashboard/src/ui/pages/AgentModule/AgentModuleTag.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentModule/CreateModal.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentModule/config.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentModule/index.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentPackage/AgentPackageTag.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentPackage/CreateModal.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentPackage/config.tsx
create mode 100644 inlong-dashboard/src/ui/pages/AgentPackage/index.tsx
delete mode 100644 inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/DefaultManagerIpListParser.java
delete mode 100644 inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/IManagerIpListParser.java
delete mode 100644 inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/pojo/DataType.java
create mode 100644 inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/pojo/InLongMetaConfig.java
create mode 100644 inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/consts/SourceConstants.java
delete mode 100644 inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SourceConstants.java
create mode 100644 inlong-distribution/src/main/assemblies/sort-connectors-v1.18.xml
delete mode 100644 inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/AuditSourceType.java
create mode 100644 inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ModuleType.java
delete mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/AuditSourceEntity.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ClusterConfigEntity.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ModuleConfigEntity.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/PackageConfigEntity.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/SortConfigEntity.java
delete mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/AuditSourceEntityMapper.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/ClusterConfigEntityMapper.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/ModuleConfigEntityMapper.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/PackageConfigEntityMapper.java
create mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/SortConfigEntityMapper.java
delete mode 100644 inlong-manager/manager-dao/src/main/resources/mappers/AuditSourceEntityMapper.xml
create mode 100644 inlong-manager/manager-dao/src/main/resources/mappers/ClusterConfigMapper.xml
create mode 100644 inlong-manager/manager-dao/src/main/resources/mappers/ModuleConfigMapper.xml
create mode 100644 inlong-manager/manager-dao/src/main/resources/mappers/PackageConfigMapper.xml
create mode 100644 inlong-manager/manager-dao/src/main/resources/mappers/SortConfigEntityMapper.xml
create mode 100644 inlong-manager/manager-plugins/manager-plugins-flink-v1.18/pom.xml
copy inlong-manager/manager-plugins/{manager-plugins-flink-v1.15 => manager-plugins-flink-v1.18}/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkClientService.java (100%)
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/audit/AuditResponse.java
delete mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/audit/AuditSourceRequest.java
delete mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/audit/AuditSourceResponse.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/cluster/InlongClusterTagExtParam.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/common/BatchResult.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/GroupFullInfo.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/ModuleDTO.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/ModuleHistory.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/ModulePageRequest.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/ModuleRequest.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/ModuleResponse.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/PackageHistory.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/PackagePageRequest.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/PackageRequest.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/module/PackageResponse.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/kafka/KafkaDataNodeDTO.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/kafka/KafkaDataNodeInfo.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/kafka/KafkaDataNodeRequest.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/DataAddTaskDTO.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/DataAddTaskRequest.java
delete mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/SubSourceDTO.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileDataAddTaskRequest.java
create mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/workflow/form/process/ClusterResourceProcessForm.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/AbstractAuditSourceOperator.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/AuditRunnable.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/AuditSourceClickhouseOperator.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/AuditSourceElasticsearchOperator.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/AuditSourceMysqlOperator.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/InlongAuditSourceOperator.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/audit/InlongAuditSourceOperatorFactory.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/InlongClusterProcessService.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/CsvDataTypeOperator.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/DataTypeOperator.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/DataTypeOperatorFactory.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/ClusterTaskListenerFactory.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/cluster/InitClusterCompleteListener.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/cluster/InitClusterFailedListener.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/cluster/InitClusterListener.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/ClusterConfigListener.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/ClusterQueueResourceListener.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/module/ModuleService.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/module/ModuleServiceImpl.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/module/PackageService.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/module/PackageServiceImpl.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/kafka/KafkaDataNodeOperator.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/QueryCountDownLatch.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/pulsar/QueryLatestMessagesRunnable.java
delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sink/ck/ClickHouseConfig.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/SortFlinkConfigOperator.java
create mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/cluster/SyncClusterResourceWorkflowDefinition.java
delete mode 100644 inlong-manager/manager-service/src/test/java/org/apache/inlong/manager/service/core/impl/AuditServiceTest.java
create mode 100644 inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/ModuleController.java
create mode 100644 inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/PackageController.java
create mode 100644 inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/openapi/InstallerController.java
create mode 100644 inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/web/controller/HeartbeatControllerTest.java
create mode 100644 inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/ClusterOperateListener.java
create mode 100644 inlong-sdk/transform-sdk/pom.xml
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonNode.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvSourceData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvSourceDecoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvUtils.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbNode.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbSourceData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbSourceDecoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceDecoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SplitUtils.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/TransformException.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/DefaultSinkData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/EscapeUtils.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkData.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/CsvSinkInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/CsvSourceInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/JsonSourceInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/KvSinkInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/KvSourceInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/PbSourceInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/ProtocolType.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/SinkInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/SourceInfo.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/TransformConfig.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ConcatFunction.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/NowFunction.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/ExpressionOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotEqualsToOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OrOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/ParenthesisOperator.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/AdditionParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/ColumnParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/DivisionParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/LongParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/MultiplicationParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/ParenthesisParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/StringParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/SubtractionParser.java
create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/ValueParser.java
create mode 100644 inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/pojo/TestTransformConfig.java
create mode 100644 inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
delete mode 100644 inlong-sort-standalone/sort-standalone-common/src/main/java/org/apache/inlong/sort/standalone/config/pojo/type/DataType.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.13/src/test/java/org/apache/inlong/sort/tests/ClickHouseITCase.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.13/src/test/java/org/apache/inlong/sort/tests/KafkaE2EITCase.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.13/src/test/java/org/apache/inlong/sort/tests/Mysql2ClickHouseTest.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.13/src/test/java/org/apache/inlong/sort/tests/Mysql2KafkaTest.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/Kafka2StarRocksTest.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/KafkaE2EITCase.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/Mongodb2StarRocksTest.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/MongodbToStarRocksITCase.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/Mysql2StarRocksTest.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/MysqlToRocksITCase.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/Postgres2StarRocksTest.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/PostgresToStarRocksITCase.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/RedisToRedisTest.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/Sqlserver2StarRocksTest.java
delete mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/SqlserverToStarRocksITCase.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE11.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE8.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/utils/RedisContainer.java
create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/resources/flinkSql/redis_test.sql
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/pom.xml
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/RedisDataType.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/RedisLookupOptions.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/RedisOptions.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/SchemaMappingMode.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/handler/FlinkJedisClusterConfigHandler.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/handler/FlinkJedisSentinelConfigHandler.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/config/handler/FlinkJedisStandaloneConfigHandler.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/container/InlongRedisClusterContainer.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/container/InlongRedisCommandsContainer.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/container/InlongRedisContainer.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/container/RedisCommandsContainerBuilder.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/descriptor/InlongRedisValidator.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/handler/InlongJedisConfigHandler.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/handler/RedisMapperHandler.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/RedisCommand.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/RedisCommandDescription.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/RedisMapper.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/row/GetMapper.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/row/HgetMapper.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/row/RowRedisMapper.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/row/ZrevrankMapper.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/mapper/row/ZscoreMapper.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/RedisSchema.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/RedisSchemaFactory.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/StateEncoder.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/impl/AbstractRedisSchema.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/impl/BitmapStaticKvPairSchema.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/impl/HashDynamicSchema.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/impl/HashStaticKvPairSchema.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/impl/HashStaticPrefixMatchSchema.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/common/schema/impl/PlainPrefixMatchSchema.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/sink/AbstractRedisSinkFunction.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/sink/RedisBitmapSinkFunction.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/sink/RedisDynamicTableSink.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/sink/RedisHashSinkFunction.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/sink/RedisPlainSinkFunction.java (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/source/RedisDynamicTableSource.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/source/RedisRowDataLookupFunction.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/table/RedisDynamicTableFactory.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/java/org/apache/inlong/sort/redis/table/SchemaValidator.java (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/resources/META-INF/services/org.apache.flink.streaming.connectors.redis.common.hanlder.RedisHandler (100%)
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.15}/sort-connectors/redis/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/test/java/org/apache/inlong/sort/redis/RedisDynamicTableFactoryTest.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/redis/src/test/java/org/apache/inlong/sort/redis/RedisTableTest.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/pom.xml
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/pom.xml
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/GenericJdbcSinkFunction.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/JdbcOutputFormat.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/TableJdbcUpsertOutputFormat.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/table/JdbcDynamicTableFactory.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/table/JdbcDynamicTableSink.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/table/JdbcOutputFormatBuilder.java
copy inlong-sort/sort-flink/{sort-flink-v1.13 => sort-flink-v1.18}/sort-connectors/jdbc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory (100%)
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pom.xml
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/pom.xml
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/PulsarTableFactory.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/PulsarTableOptionUtils.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/PulsarTableOptions.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/PulsarTableValidationUtils.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/source/PulsarTableDeserializationSchema.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/source/PulsarTableDeserializationSchemaFactory.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/java/org/apache/inlong/sort/pulsar/table/source/PulsarTableSource.java
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-connectors/pulsar/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
create mode 100644 inlong-sort/sort-flink/sort-flink-v1.18/sort-flink-dependencies/pom.xml
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/ArrayFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/ArrayTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/BasicFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/BinaryFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/BinaryTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/BooleanFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/BooleanTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/ByteFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/ByteTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/Constants.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/DateFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/DateTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/DecimalFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/DecimalTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/DoubleFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/DoubleTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/FloatFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/FloatTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/FormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/FormatUtils.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/IntFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/IntTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/LocalZonedTimestampFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/LocalZonedTimestampTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/LongFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/LongTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/MapFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/MapTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/NullFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/NullTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/ShortFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/ShortTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/StringFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/StringTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/TimeFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/TimeTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/TimestampFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/TimestampTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/TypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/VarBinaryFormatInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/VarBinaryTypeInfo.java
delete mode 100644 inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/VarCharFormatInfo.java
create mode 100644 inlong-sort/sort-formats/format-row/format-json-v1.18/pom.xml
create mode 100644 inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/MysqlBinLogData.java
copy inlong-sort/sort-formats/format-row/{format-json-v1.15 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJson.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonDecodingFormat.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonDeserializationSchema.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedDecodingFormat.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedDeserializationSchema.java (100%)
create mode 100644 inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedEncodingFormat.java
copy inlong-sort/sort-formats/format-row/{format-json-v1.15 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedFormatFactory.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.15 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerializationSchema.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonSerializationSchema.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalUtils.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/debezium/DebeziumJson.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/debezium/DebeziumJsonDecodingFormat.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/debezium/DebeziumJsonDeserializationSchema.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/java/org/apache/inlong/sort/formats/json/debezium/DebeziumUtils.java (100%)
create mode 100644 inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/utils/FormatJsonUtil.java
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.15 => format-json-v1.18}/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedFormatFactoryTest.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.15 => format-json-v1.18}/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerDeSchemaTest.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonSerializationTest.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/test/java/org/apache/inlong/sort/formats/json/canal/DebeziumJsonSerializationTest.java (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/test/resources/canal-json-inlong-data.txt (100%)
copy inlong-sort/sort-formats/format-row/{format-json-v1.13 => format-json-v1.18}/src/test/resources/log4j2-test.properties (100%)
(inlong) 04/04: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit dadc2d194eab39e412faac6c0a0ee193afc692b5
Author: AloysZhang <al...@apache.org>
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ++++---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +++++
.../listener/StreamTaskListenerFactory.java | 28 +++++++
.../schedule/StreamScheduleResourceListener.java | 95 ++++++++++++++++++++++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36 ++++----
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+ public static final String REGISTER_SCHEDULE_STATUS = "register.schedule.status";
+
+ public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+ boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+ .equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+ // do not submit flink job if the group mode is offline and the stream is not config successfully
+ if (isOfflineSync && !StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+ log.info("no need to submit flink job for groupId={} streamId={} as the mode is offline "
+ + "and the stream is not config successfully yet", groupId, streamInfo.getInlongStreamId());
+ continue;
+ }
List<StreamSink> sinkList = streamInfo.getSinkList();
List<String> sinkTypes = sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) || !SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements SortOperateListener {
return ListenerResult.fail(message);
}
- boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
- .equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName = Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
- if (isRealTimeSync) {
- flinkInfo.setRuntimeExecutionMode(InlongConstants.RUNTIME_EXECUTION_MODE_STREAMING);
- } else {
+ if (isOfflineSync) {
flinkInfo.setRuntimeExecutionMode(InlongConstants.RUNTIME_EXECUTION_MODE_BATCH);
+ } else {
+ flinkInfo.setRuntimeExecutionMode(InlongConstants.RUNTIME_EXECUTION_MODE_STREAMING);
}
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
- // only start job for real-time mode
- if (isRealTimeSync) {
- flinkOperation.start(flinkInfo);
- log.info("job submit success for groupId = {}, streamId = {}, jobId = {}", groupId,
- streamInfo.getInlongStreamId(), flinkInfo.getJobId());
- }
+ flinkOperation.start(flinkInfo);
+ log.info("job submit success for groupId = {}, mode = {}, streamId = {}, jobId = {}",
+ groupId, groupResourceForm.getGroupInfo().getInlongGroupMode(),
+ streamInfo.getInlongStreamId(), flinkInfo.getJobId());
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
index 92470968ac..203bcc6778 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
@@ -17,11 +17,14 @@
package org.apache.inlong.manager.pojo.sort.util;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.enums.TransformType;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.pojo.sink.StreamSink;
import org.apache.inlong.manager.pojo.source.StreamSource;
+import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.pojo.stream.StreamNode;
import org.apache.inlong.manager.pojo.stream.StreamPipeline;
import org.apache.inlong.manager.pojo.stream.StreamTransform;
@@ -38,6 +41,8 @@ import org.apache.inlong.manager.pojo.transform.splitter.SplitterDefinition;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
/**
* Utils of stream parse.
@@ -154,4 +159,20 @@ public class StreamParseUtils {
return GSON.fromJson(tempView, StreamPipeline.class);
}
+ public static String getStreamExtProperty(String key, InlongStreamInfo streamInfo) {
+ if (StringUtils.isNotBlank(key) && streamInfo != null && CollectionUtils.isNotEmpty(streamInfo.getExtList())) {
+ for (InlongStreamExtInfo ext : streamInfo.getExtList()) {
+ if (key.equalsIgnoreCase(ext.getKeyName())) {
+ return ext.getKeyValue();
+ }
+ }
+ }
+ return null;
+ }
+
+ public static boolean isRegisterScheduleSuccess(InlongStreamInfo streamInfo) {
+ return InlongConstants.REGISTERED
+ .equalsIgnoreCase(getStreamExtProperty(InlongConstants.REGISTER_SCHEDULE_STATUS, streamInfo));
+ }
+
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
index 43fdfe4666..87c3a867c2 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
@@ -20,12 +20,14 @@ package org.apache.inlong.manager.service.listener;
import org.apache.inlong.manager.common.plugin.Plugin;
import org.apache.inlong.manager.common.plugin.PluginBinder;
import org.apache.inlong.manager.service.listener.queue.StreamQueueResourceListener;
+import org.apache.inlong.manager.service.listener.schedule.StreamScheduleResourceListener;
import org.apache.inlong.manager.service.listener.sink.StreamSinkResourceListener;
import org.apache.inlong.manager.service.listener.sort.StreamSortConfigListener;
import org.apache.inlong.manager.workflow.WorkflowContext;
import org.apache.inlong.manager.workflow.definition.ServiceTaskType;
import org.apache.inlong.manager.workflow.definition.TaskListenerFactory;
import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
+import org.apache.inlong.manager.workflow.event.task.ScheduleOperateListener;
import org.apache.inlong.manager.workflow.event.task.SinkOperateListener;
import org.apache.inlong.manager.workflow.event.task.SortOperateListener;
import org.apache.inlong.manager.workflow.event.task.SourceOperateListener;
@@ -53,6 +55,7 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
private List<QueueOperateListener> queueOperateListeners;
private List<SortOperateListener> sortOperateListeners;
private List<SinkOperateListener> sinkOperateListeners;
+ private List<ScheduleOperateListener> scheduleOperateListeners;
@Autowired
private StreamQueueResourceListener queueResourceListener;
@@ -60,6 +63,8 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
private StreamSortConfigListener streamSortConfigListener;
@Autowired
private StreamSinkResourceListener sinkResourceListener;
+ @Autowired
+ private StreamScheduleResourceListener scheduleResourceListener;
@PostConstruct
public void init() {
@@ -70,6 +75,8 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
sortOperateListeners.add(streamSortConfigListener);
sinkOperateListeners = new LinkedList<>();
sinkOperateListeners.add(sinkResourceListener);
+ scheduleOperateListeners = new LinkedList<>();
+ scheduleOperateListeners.add(scheduleResourceListener);
}
@Override
@@ -94,6 +101,10 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
if (CollectionUtils.isNotEmpty(pluginSinkOperateListeners)) {
sinkOperateListeners.addAll(pluginSinkOperateListeners);
}
+ List<ScheduleOperateListener> pluginScheduleOperateListeners = processPlugin.createScheduleOperateListeners();
+ if (CollectionUtils.isNotEmpty(pluginScheduleOperateListeners)) {
+ scheduleOperateListeners.addAll(pluginScheduleOperateListeners);
+ }
}
@Override
@@ -118,6 +129,9 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
case INIT_SINK:
List<SinkOperateListener> sinkOperateListeners = getSinkOperateListener(workflowContext);
return Lists.newArrayList(sinkOperateListeners);
+ case INIT_SCHEDULE:
+ List<ScheduleOperateListener> scheduleOperateListeners = getScheduleOperateListener(workflowContext);
+ return Lists.newArrayList(scheduleOperateListeners);
default:
throw new IllegalArgumentException(String.format("Unsupported ServiceTaskType %s", taskType));
}
@@ -131,6 +145,7 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
queueOperateListeners = new LinkedList<>();
sortOperateListeners = new LinkedList<>();
sinkOperateListeners = new LinkedList<>();
+ scheduleOperateListeners = new LinkedList<>();
}
/**
@@ -185,4 +200,17 @@ public class StreamTaskListenerFactory implements PluginBinder, TaskListenerFact
return listeners;
}
+ /**
+ * Get schedule operate listener list.
+ */
+ private List<ScheduleOperateListener> getScheduleOperateListener(WorkflowContext context) {
+ List<ScheduleOperateListener> listeners = new ArrayList<>();
+ for (ScheduleOperateListener listener : scheduleOperateListeners) {
+ if (listener != null && listener.accept(context)) {
+ listeners.add(listener);
+ }
+ }
+ return listeners;
+ }
+
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/schedule/StreamScheduleResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/schedule/StreamScheduleResourceListener.java
new file mode 100644
index 0000000000..61179dc343
--- /dev/null
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/schedule/StreamScheduleResourceListener.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.listener.schedule;
+
+import org.apache.inlong.manager.common.consts.InlongConstants;
+import org.apache.inlong.manager.common.enums.GroupOperateType;
+import org.apache.inlong.manager.common.enums.TaskEvent;
+import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.pojo.workflow.form.process.ProcessForm;
+import org.apache.inlong.manager.pojo.workflow.form.process.StreamResourceProcessForm;
+import org.apache.inlong.manager.workflow.WorkflowContext;
+import org.apache.inlong.manager.workflow.event.ListenerResult;
+import org.apache.inlong.manager.workflow.event.task.ScheduleOperateListener;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+@Slf4j
+public class StreamScheduleResourceListener implements ScheduleOperateListener {
+
+ @Override
+ public TaskEvent event() {
+ return TaskEvent.COMPLETE;
+ }
+
+ @Override
+ public boolean accept(WorkflowContext context) {
+ ProcessForm processForm = context.getProcessForm();
+ String groupId = processForm.getInlongGroupId();
+ if (!(processForm instanceof StreamResourceProcessForm)) {
+ log.info("not add schedule stream listener, not StreamResourceProcessForm for groupId [{}]", groupId);
+ return false;
+ }
+
+ StreamResourceProcessForm streamProcessForm = (StreamResourceProcessForm) processForm;
+ String streamId = streamProcessForm.getStreamInfo().getInlongStreamId();
+ if (streamProcessForm.getGroupOperateType() != GroupOperateType.INIT) {
+ log.info("not add schedule stream listener, as the operate was not INIT for groupId [{}] streamId [{}]",
+ groupId, streamId);
+ return false;
+ }
+
+ log.info("add startup stream listener for groupId [{}] streamId [{}]", groupId, streamId);
+ return InlongConstants.DATASYNC_OFFLINE_MODE.equals(streamProcessForm.getGroupInfo().getInlongGroupMode());
+ }
+
+ @Override
+ public ListenerResult listen(WorkflowContext context) throws Exception {
+ StreamResourceProcessForm form = (StreamResourceProcessForm) context.getProcessForm();
+ InlongStreamInfo streamInfo = form.getStreamInfo();
+ final String groupId = streamInfo.getInlongGroupId();
+ final String streamId = streamInfo.getInlongStreamId();
+ log.info("begin to register schedule info for groupId={}, streamId={}", groupId, streamId);
+
+ // todo: register schedule info to schedule service
+
+ // after register schedule info successfully, add ext property to stream info
+ saveInfo(streamInfo, InlongConstants.REGISTER_SCHEDULE_STATUS,
+ InlongConstants.REGISTERED, streamInfo.getExtList());
+ log.info("success to register schedule info for group [" + groupId + "] and stream [" + streamId + "]");
+ return ListenerResult.success();
+ }
+
+ /**
+ * Save stream ext info into list.
+ */
+ private void saveInfo(InlongStreamInfo streamInfo, String keyName, String keyValue,
+ List<InlongStreamExtInfo> extInfoList) {
+ InlongStreamExtInfo extInfo = new InlongStreamExtInfo();
+ extInfo.setInlongGroupId(streamInfo.getInlongGroupId());
+ extInfo.setInlongStreamId(streamInfo.getInlongStreamId());
+ extInfo.setKeyName(keyName);
+ extInfo.setKeyValue(keyValue);
+ extInfoList.add(extInfo);
+ }
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java
index ebed9e72e3..f8981bf09a 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/sort/SortConfigListener.java
@@ -24,6 +24,7 @@ import org.apache.inlong.manager.common.enums.TaskEvent;
import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
import org.apache.inlong.manager.pojo.workflow.form.process.ProcessForm;
@@ -125,6 +126,13 @@ public class SortConfigListener implements SortOperateListener {
try {
for (InlongStreamInfo streamInfo : streamInfos) {
+ // do not build sort config if the group mode is offline and the stream is not config successfully
+ if (InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())
+ && !StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+ LOGGER.info("no need to build sort config for groupId={} streamId={} as the mode is offline "
+ + "and the stream is not config successfully yet", groupId, streamInfo.getInlongStreamId());
+ continue;
+ }
List<StreamSink> sinkList = streamInfo.getSinkList();
if (CollectionUtils.isEmpty(sinkList)) {
continue;
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java
index d3cf199a21..5505ed0694 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/workflow/stream/CreateStreamWorkflowDefinition.java
@@ -102,17 +102,26 @@ public class CreateStreamWorkflowDefinition implements WorkflowDefinition {
initSourceTask.setListenerFactory(streamTaskListenerFactory);
process.addTask(initSourceTask);
+ // Init Schedule info
+ ServiceTask initScheduleTask = new ServiceTask();
+ initScheduleTask.setName("InitSchedule");
+ initScheduleTask.setDisplayName("Stream-InitSchedule");
+ initScheduleTask.setServiceTaskType(ServiceTaskType.INIT_SCHEDULE);
+ initScheduleTask.setListenerFactory(streamTaskListenerFactory);
+ process.addTask(initScheduleTask);
+
// End node
EndEvent endEvent = new EndEvent();
process.setEndEvent(endEvent);
- // Task dependency order: 1.MQ -> 2.Sink -> 3.Sort -> 4.Source
+ // Task dependency order: 1.MQ -> 2.Sink -> 3.Sort -> 4.Source -> 5.Schedule
// To ensure that after some tasks fail, data will not start to be collected by source or consumed by sort
startEvent.addNext(initMQTask);
initMQTask.addNext(initSinkTask);
initSinkTask.addNext(initSortTask);
initSortTask.addNext(initSourceTask);
- initSourceTask.addNext(endEvent);
+ initSourceTask.addNext(initScheduleTask);
+ initScheduleTask.addNext(endEvent);
return process;
}
diff --git a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java
index 2686b4b60a..156863a676 100644
--- a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java
+++ b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java
@@ -25,6 +25,7 @@ public enum ServiceTaskType {
INIT_MQ,
INIT_SORT,
INIT_SINK,
+ INIT_SCHEDULE,
STOP_SOURCE,
STOP_SORT,
diff --git a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/ScheduleOperateListener.java
similarity index 55%
copy from inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java
copy to inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/ScheduleOperateListener.java
index 2686b4b60a..c3e162cd26 100644
--- a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/definition/ServiceTaskType.java
+++ b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/event/task/ScheduleOperateListener.java
@@ -15,30 +15,24 @@
* limitations under the License.
*/
-package org.apache.inlong.manager.workflow.definition;
+package org.apache.inlong.manager.workflow.event.task;
-import java.util.Locale;
+import org.apache.inlong.manager.common.enums.TaskEvent;
+import org.apache.inlong.manager.workflow.WorkflowContext;
+import org.apache.inlong.manager.workflow.event.ListenerResult;
-public enum ServiceTaskType {
+public interface ScheduleOperateListener extends TaskEventListener {
- INIT_SOURCE,
- INIT_MQ,
- INIT_SORT,
- INIT_SINK,
+ ScheduleOperateListener DEFAULT_SCHEDULE_OPERATE_LISTENER = new ScheduleOperateListener() {
- STOP_SOURCE,
- STOP_SORT,
-
- RESTART_SOURCE,
- RESTART_SORT,
-
- DELETE_SOURCE,
- DELETE_MQ,
- DELETE_SORT;
-
- @Override
- public String toString() {
- return name().toLowerCase(Locale.ROOT);
- }
+ @Override
+ public TaskEvent event() {
+ return TaskEvent.COMPLETE;
+ }
+ @Override
+ public ListenerResult listen(WorkflowContext context) {
+ return ListenerResult.success();
+ }
+ };
}
diff --git a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/plugin/ProcessPlugin.java b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/plugin/ProcessPlugin.java
index 7e15d52761..452fa2bfb8 100644
--- a/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/plugin/ProcessPlugin.java
+++ b/inlong-manager/manager-workflow/src/main/java/org/apache/inlong/manager/workflow/plugin/ProcessPlugin.java
@@ -19,6 +19,7 @@ package org.apache.inlong.manager.workflow.plugin;
import org.apache.inlong.manager.common.plugin.Plugin;
import org.apache.inlong.manager.workflow.event.task.QueueOperateListener;
+import org.apache.inlong.manager.workflow.event.task.ScheduleOperateListener;
import org.apache.inlong.manager.workflow.event.task.SinkOperateListener;
import org.apache.inlong.manager.workflow.event.task.SortOperateListener;
import org.apache.inlong.manager.workflow.event.task.SourceOperateListener;
@@ -38,6 +39,10 @@ public interface ProcessPlugin extends Plugin {
return null;
}
+ default List<ScheduleOperateListener> createScheduleOperateListeners() {
+ return null;
+ }
+
default List<QueueOperateListener> createQueueOperateListeners() {
return null;
}
(inlong) 01/04: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit cfd9136286502a4f33d3482461218f814f4f2a6a
Author: AloysZhang <al...@apache.org>
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++++++++++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 ++++--
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java | 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java | 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java | 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
--- a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++ b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
- pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+ pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(10000000);
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
- public static final Integer DATASYNC_MODE = 1;
+ public static final Integer DATASYNC_REALTIME_MODE = 1;
+ public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
- DATASYNC("datasync");
+ DATASYNC("datasync"),
+
+ /**
+ * DataSync mode(only Data Synchronization): offline data sync in batch way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+ DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,7 @@ public class StartupSortListener implements SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
- return InlongConstants.DATASYNC_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+ return InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
}
@Override
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
index 5b3e730ed9..17b66497a9 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
@@ -72,7 +72,8 @@ public abstract class InlongGroupInfo extends BaseInlongGroup {
@ApiModelProperty(value = "Whether to enable create resource? 0: disable, 1: enable")
private Integer enableCreateResource;
- @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1")
+ @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization, real-time data sync in stream way): 1,"
+ + " DataSync mode(only Data Synchronization, offline data sync in batch way): 2")
private Integer inlongGroupMode;
@ApiModelProperty(value = "Data report type, default is 0.\n"
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
index 5d7c24b295..4dc2140166 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
@@ -58,7 +58,8 @@ public class InlongGroupPageRequest extends PageRequest {
@ApiModelProperty(value = "The inlong cluster tag list")
private List<String> clusterTagList;
- @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1")
+ @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization, real-time data sync in stream way): 1,"
+ + " DataSync mode(only Data Synchronization, offline data sync in batch way): 2")
private Integer inlongGroupMode;
@ApiModelProperty(value = "Current user", hidden = true)
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
index 30893b7a09..6140bddad5 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
@@ -86,8 +86,9 @@ public abstract class InlongGroupRequest extends BaseInlongGroup {
@Range(min = 0, max = 1, message = "default is 1, only supports [0: disable, 1: enable]")
private Integer enableCreateResource;
- @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1")
- @Range(min = 0, max = 1, message = "default is 0, only supports [0: Standard, 1: DataSync]")
+ @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization, real-time data sync in stream way): 1,"
+ + " DataSync mode(only Data Synchronization, offline data sync in batch way): 2")
+ @Range(min = 0, max = 2, message = "default is 0, only supports [0: Standard, 1: DataSync, 2: DataSyncOffline]")
private Integer inlongGroupMode;
@ApiModelProperty(value = "Data report type, default is 0.\n"
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
index 8230771b76..da24563475 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
@@ -209,7 +209,8 @@ public class AuditServiceImpl implements AuditService {
if (CollectionUtils.isEmpty(request.getAuditIds())) {
// properly overwrite audit ids by role and stream config
- if (InlongConstants.DATASYNC_MODE.equals(groupEntity.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupEntity.getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupEntity.getInlongGroupMode())) {
auditIdMap.put(getAuditId(sourceNodeType, IndicatorType.RECEIVED_SUCCESS), sourceNodeType);
request.setAuditIds(getAuditIds(groupId, streamId, sourceNodeType, sinkNodeType));
} else {
@@ -320,7 +321,8 @@ public class AuditServiceImpl implements AuditService {
} else {
auditSet.add(getAuditId(sinkNodeType, IndicatorType.SEND_SUCCESS));
InlongGroupEntity inlongGroup = inlongGroupMapper.selectByGroupId(groupId);
- if (InlongConstants.DATASYNC_MODE.equals(inlongGroup.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(inlongGroup.getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(inlongGroup.getInlongGroupMode())) {
auditSet.add(getAuditId(sourceNodeType, IndicatorType.RECEIVED_SUCCESS));
} else {
auditSet.add(getAuditId(sinkNodeType, IndicatorType.RECEIVED_SUCCESS));
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
index 49e55cfbd0..78257b35d9 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
@@ -731,7 +731,7 @@ public class InlongGroupServiceImpl implements InlongGroupService {
InlongGroupInfo groupInfo = this.get(groupId);
// check if the group mode is data sync mode
- if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
String errMSg = String.format("no need to switch sync mode group = {}", groupId);
LOGGER.error(errMSg);
throw new BusinessException(errMSg);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index db8cffbbe1..c96c12c9ff 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,7 @@ public class InitGroupCompleteListener implements ProcessEventListener {
// update status of other related configs
if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource())) {
- if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null, SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null, SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index 445ee647ef..a6dd8fc786 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
@@ -96,7 +96,7 @@ public class UpdateGroupCompleteListener implements ProcessEventListener {
}
// if the inlong group is dataSync mode, the stream source needs to be processed.
- if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
changeSource4DataSync(groupId, operateType, operator);
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
index 79eb6638cc..f566c29728 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
@@ -115,7 +115,7 @@ public class QueueResourceListener implements QueueOperateListener {
String operator = context.getOperator();
GroupOperateType operateType = groupProcessForm.getGroupOperateType();
- if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
log.warn("skip to execute QueueResourceListener as sync mode for groupId={}", groupId);
if (GroupOperateType.INIT.equals(operateType)) {
this.createQueueForStreams(groupInfo, groupProcessForm.getStreamInfos(), operator);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
index a739c5da02..df41823b84 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
@@ -66,7 +66,7 @@ public class InitStreamCompleteListener implements ProcessEventListener {
// Update status of other related configs
streamService.updateStatus(groupId, streamId, StreamStatus.CONFIG_SUCCESSFUL.getCode(), operator);
streamService.updateWithoutCheck(streamInfo.genRequest(), operator);
- if (InlongConstants.DATASYNC_MODE.equals(form.getGroupInfo().getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(form.getGroupInfo().getInlongGroupMode())) {
sourceService.updateStatus(groupId, streamId, SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, streamId, SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java
index 9428101c40..83b7ac3ea9 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java
@@ -138,7 +138,7 @@ public abstract class AbstractSourceOperator implements StreamSourceOperator {
updateFieldOpt(entity, request.getFieldList());
return;
}
- boolean allowUpdate = InlongConstants.DATASYNC_MODE.equals(groupMode)
+ boolean allowUpdate = InlongConstants.DATASYNC_REALTIME_MODE.equals(groupMode)
|| SourceStatus.ALLOWED_UPDATE.contains(entity.getStatus());
if (!allowUpdate) {
throw new BusinessException(ErrorCodeEnum.SOURCE_OPT_NOT_ALLOWED,
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
index c252a6a00c..3dd8fa5f26 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
@@ -248,7 +248,7 @@ public class StreamSourceServiceImpl implements StreamSourceService {
// if the group mode is DATASYNC, just get all related stream sources
List<StreamSource> streamSources = this.listSource(groupId, null);
- if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
result = streamSources.stream()
.collect(Collectors.groupingBy(StreamSource::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
(inlong) 03/04: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit ddc72ed417cbd12f06f463d646bf6ed841ffd67d
Author: AloysZhang <al...@apache.org>
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 2 ++
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++++++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++++++++++----
.../org/apache/inlong/sort/configuration/Constants.java | 8 ++++++++
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++++++++++-
8 files changed, 43 insertions(+), 6 deletions(-)
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+ public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+ public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 69a32b73fa..33a7ca59cd 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,8 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("60000");
+ list.add("-runtime.execution.mode");
+ list.add(flinkInfo.getRuntimeExecutionMode());
list.add("-metrics.audit.proxy.hosts");
list.add(flinkConfig.getAuditProxyHosts());
return list.toArray(new String[0]);
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+ private String runtimeExecutionMode;
}
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+ // TODO if the job is OFFLINE, should delete the scheduler information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
@@ -86,7 +86,13 @@ public class RestartSortListener implements SortOperateListener {
}
GroupResourceProcessForm groupResourceProcessForm = (GroupResourceProcessForm) processForm;
-
+ if (InlongConstants.DATASYNC_OFFLINE_MODE.equals(
+ groupResourceProcessForm.getGroupInfo().getInlongGroupMode())) {
+ String message = String.format("offline data sync job should be scheduled by the "
+ + "scheduler system only for groupId [%s]", groupId);
+ log.error(message);
+ return ListenerResult.fail(message);
+ }
List<InlongStreamInfo> streamInfos = groupResourceProcessForm.getStreamInfos();
for (InlongStreamInfo streamInfo : streamInfos) {
List<StreamSink> sinkList = streamInfo.getSinkList();
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index fae5faa278..f6b3b6061a 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -131,20 +131,26 @@ public class StartupSortListener implements SortOperateListener {
return ListenerResult.fail(message);
}
+ boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
+ .equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+
FlinkInfo flinkInfo = new FlinkInfo();
String jobName = Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
+ streamInfo.getInlongStreamId();
flinkInfo.setJobName(jobName);
- String sortUrl = kvConf.get(InlongConstants.SORT_URL);
- flinkInfo.setEndpoint(sortUrl);
+ flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
+ if (isRealTimeSync) {
+ flinkInfo.setRuntimeExecutionMode(InlongConstants.RUNTIME_EXECUTION_MODE_STREAMING);
+ } else {
+ flinkInfo.setRuntimeExecutionMode(InlongConstants.RUNTIME_EXECUTION_MODE_BATCH);
+ }
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
// only start job for real-time mode
- if (InlongConstants.DATASYNC_REALTIME_MODE
- .equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+ if (isRealTimeSync) {
flinkOperation.start(flinkInfo);
log.info("job submit success for groupId = {}, streamId = {}, jobId = {}", groupId,
streamInfo.getInlongStreamId(), flinkInfo.getJobId());
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/configuration/Constants.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/configuration/Constants.java
index 702dbef53f..a5717242a5 100644
--- a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/configuration/Constants.java
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/configuration/Constants.java
@@ -291,6 +291,14 @@ public class Constants {
public static final ConfigOption<Integer> CHECKPOINT_TIMEOUT_MS =
key("checkpoint.timeout.ms").defaultValue(600000);
+ // ------------------------------------------------------------------------
+ // Flink runtime execution mode, including stream and batch, default is stream
+ // ------------------------------------------------------------------------
+
+ public static final ConfigOption<String> RUNTIME_EXECUTION_MODE = key("runtime.execution.mode")
+ .defaultValue("stream")
+ .withDescription("The runtime execution mode of Flink, including stream and batch, default is stream");
+
// ------------------------------------------------------------------------
// Metrics related
// ------------------------------------------------------------------------
diff --git a/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/Entrance.java b/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/Entrance.java
index ea3f3f95be..4aa3902cae 100644
--- a/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/Entrance.java
+++ b/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/Entrance.java
@@ -40,6 +40,8 @@ import java.nio.charset.StandardCharsets;
public class Entrance {
+ public static final String BATCH_MODE = "batch";
+
public static void main(String[] args) throws Exception {
final ParameterTool parameterTool = ParameterTool.fromArgs(args);
final Configuration config = parameterTool.getConfiguration();
@@ -50,7 +52,14 @@ public class Entrance {
config.getInteger(Constants.MIN_PAUSE_BETWEEN_CHECKPOINTS_MS));
env.getCheckpointConfig().setCheckpointTimeout(config.getInteger(Constants.CHECKPOINT_TIMEOUT_MS));
env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
- EnvironmentSettings settings = EnvironmentSettings.newInstance().inStreamingMode().build();
+
+ String runtimeExecutionMode = config.getString(Constants.RUNTIME_EXECUTION_MODE);
+ EnvironmentSettings settings;
+ if (BATCH_MODE.equalsIgnoreCase(runtimeExecutionMode)) {
+ settings = EnvironmentSettings.newInstance().inBatchMode().build();
+ } else {
+ settings = EnvironmentSettings.newInstance().inStreamingMode().build();
+ }
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env, settings);
tableEnv.getConfig().getConfiguration().setString(Constants.PIPELINE_NAME,
config.getString(Constants.JOB_NAME));
(inlong) 02/04: [INLONG-9813][Manager] Support offline data sync management (#9814)
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit e66cf075fe41e45f44c9b13e250242b0aaf4e0a3
Author: AloysZhang <al...@apache.org>
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +++++++++----
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 ++++--
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
--- a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++ b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
- return InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+ return (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
- flinkOperation.start(flinkInfo);
- log.info("job submit success for groupId = {}, streamId = {}, jobId = {}", groupId,
- streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+ // only start job for real-time mode
+ if (InlongConstants.DATASYNC_REALTIME_MODE
+ .equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+ flinkOperation.start(flinkInfo);
+ log.info("job submit success for groupId = {}, streamId = {}, jobId = {}", groupId,
+ streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+ }
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements ProcessEventListener {
// update status of other related configs
if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource())) {
- if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null, SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null, SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
@@ -96,7 +96,8 @@ public class UpdateGroupCompleteListener implements ProcessEventListener {
}
// if the inlong group is dataSync mode, the stream source needs to be processed.
- if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
changeSource4DataSync(groupId, operateType, operator);
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
index f566c29728..0ce551d1e6 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
@@ -115,8 +115,10 @@ public class QueueResourceListener implements QueueOperateListener {
String operator = context.getOperator();
GroupOperateType operateType = groupProcessForm.getGroupOperateType();
- if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
- log.warn("skip to execute QueueResourceListener as sync mode for groupId={}", groupId);
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
+ log.warn("skip to execute QueueResourceListener as sync mode {} (1 for realtime sync, 2 for offline sync) "
+ + "for groupId={}", groupInfo.getInlongGroupMode(), groupId);
if (GroupOperateType.INIT.equals(operateType)) {
this.createQueueForStreams(groupInfo, groupProcessForm.getStreamInfos(), operator);
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
index df41823b84..85033b44f7 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/stream/InitStreamCompleteListener.java
@@ -66,7 +66,8 @@ public class InitStreamCompleteListener implements ProcessEventListener {
// Update status of other related configs
streamService.updateStatus(groupId, streamId, StreamStatus.CONFIG_SUCCESSFUL.getCode(), operator);
streamService.updateWithoutCheck(streamInfo.genRequest(), operator);
- if (InlongConstants.DATASYNC_REALTIME_MODE.equals(form.getGroupInfo().getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(form.getGroupInfo().getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(form.getGroupInfo().getInlongGroupMode())) {
sourceService.updateStatus(groupId, streamId, SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, streamId, SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java
index 83b7ac3ea9..39d3845209 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/AbstractSourceOperator.java
@@ -139,6 +139,7 @@ public abstract class AbstractSourceOperator implements StreamSourceOperator {
return;
}
boolean allowUpdate = InlongConstants.DATASYNC_REALTIME_MODE.equals(groupMode)
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupMode)
|| SourceStatus.ALLOWED_UPDATE.contains(entity.getStatus());
if (!allowUpdate) {
throw new BusinessException(ErrorCodeEnum.SOURCE_OPT_NOT_ALLOWED,
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
index 3dd8fa5f26..0bb15d6cad 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
@@ -248,7 +248,8 @@ public class StreamSourceServiceImpl implements StreamSourceService {
// if the group mode is DATASYNC, just get all related stream sources
List<StreamSource> streamSources = this.listSource(groupId, null);
- if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())) {
+ if (InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+ || InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
result = streamSources.stream()
.collect(Collectors.groupingBy(StreamSource::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));