You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by xi...@apache.org on 2022/03/15 08:33:59 UTC
[rocketmq-streams] branch main_develop updated (fe3cc4e -> 9332635)
This is an automated email from the ASF dual-hosted git repository.
xiaodong pushed a change to branch main_develop
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git.
from fe3cc4e [ISSUES 135] App can not start up with RocketMQ-streams (#136)
new 2db9d36 add kafka module
new f65653f add kafka module
new 4565739 add licence for some files
new 9332635 Merge pull request #137 from program-0/snapshot-1.0.2
The 411 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
NOTICE | 2 +-
README-chinese.md | 22 +-
README.md | 28 +-
docs/README.md | 8 +
...225\264\344\275\223\346\236\266\346\236\204.md" | 33 --
.../2.\346\236\204\345\273\272DataStream.md" | 73 ----
.../3.\345\220\257\345\212\250DataStream.md" | 53 ---
...265\201\350\275\254\350\277\207\347\250\213.md" | 63 ---
...256\227\345\255\220\350\247\243\346\236\220.md" | 55 ---
"docs/images/Pipeline\347\261\273\345\233\276.png" | Bin 44207 -> 0 bytes
docs/images/img.png | Bin 0 -> 38684 bytes
docs/images/img_1.png | Bin 0 -> 43711 bytes
docs/images/img_2.png | Bin 0 -> 103151 bytes
docs/images/window.png | Bin 241692 -> 0 bytes
...75\223\346\236\266\346\236\204\345\233\276.png" | Bin 60493 -> 0 bytes
...00\273\344\275\223\350\277\207\347\250\213.png" | Bin 44252 -> 0 bytes
.../\346\211\251\345\256\271\345\211\215.png" | Bin 56733 -> 0 bytes
...12\266\346\200\201\347\256\227\345\255\220.png" | Bin 35766 -> 0 bytes
"docs/images/\347\212\266\346\200\201.png" | Bin 47527 -> 0 bytes
"docs/images/\347\274\251\345\256\271.png" | Bin 51087 -> 0 bytes
docs/stream_sink/README.md | 19 +-
docs/stream_source/README.md | 30 +-
pom.xml | 89 +---
rocketmq-streams-channel-db/pom.xml | 6 +-
rocketmq-streams-channel-es/pom.xml | 27 +-
.../rocketmq/streams/es/sink/ESChannelBuilder.java | 46 +-
.../streams/es/sink/ESSinkOnlyChannel.java | 10 +-
.../apache/rocketmq/streams/es/sink/EsClient.java | 135 ++++++
rocketmq-streams-channel-http/pom.xml | 6 +-
rocketmq-streams-channel-kafka/pom.xml | 32 ++
.../streams/kafka/KafkaChannelBuilder.java | 41 +-
.../apache/rocketmq/streams/kafka/KafkaSplit.java | 26 +-
.../rocketmq/streams/kafka/sink/KafkaSink.java | 200 +++++++++
.../rocketmq/streams/kafka/source/KafkaSource.java | 238 ++++++++++
.../rocketmq/streams/kafka/KafkaChannelTest.java | 104 +++++
.../src/test/resources/log4j.xml | 20 +
rocketmq-streams-channel-mqtt/pom.xml | 22 +-
.../rocketmq/streams/mqtt/source/PahoSource.java | 28 +-
rocketmq-streams-channel-rocketmq/pom.xml | 10 +-
.../apache/rocketmq/streams/sink/RocketMQSink.java | 25 +-
.../rocketmq/streams/source/RocketMQSource.java | 14 +-
.../rocketmq/streams/RocketMQChannelTest.java | 2 +-
rocketmq-streams-channel-syslog/pom.xml | 21 +-
rocketmq-streams-checkpoint/pom.xml | 6 +-
rocketmq-streams-clients/pom.xml | 11 +-
.../streams/client/source/DataStreamSource.java | 20 +
.../streams/client/transform/DataStream.java | 21 +-
.../streams/client/transform/JoinStream.java | 5 +-
.../streams/client/transform/SplitStream.java | 24 +-
.../rocketmq/streams/client/ApplicationTest.java | 57 +++
.../rocketmq/streams/client/MqttSourceExample.java | 80 ++++
.../client/{SinkTest.java => ScriptTest.java} | 24 +-
.../rocketmq/streams/client/example/JoinTest.java | 7 +-
.../example/{DimTest.java => SplitTest.java} | 43 +-
.../streams/client/sink/UserDefinedSink.java | 2 +-
.../client/sink/UserDefinedSupportShuffleSink.java | 4 +-
rocketmq-streams-commons/pom.xml | 31 +-
.../MappedByteBufferTableWithPrimaryIndex.java | 482 +++++++++++++++++++++
.../streams/common/cache/compress/KVAddress.java | 48 +-
.../streams/common/channel/AbstractChannel.java | 3 +-
.../common/channel/builder/IChannelBuilder.java | 25 +-
...OutputPrintChannel.java => CollectionSink.java} | 31 +-
.../common/channel/impl/CollectionSinkBuilder.java | 30 +-
.../common/channel/impl/PrintChannelBuilder.java | 26 +-
.../channel/impl/file/FileChannelBuilder.java | 45 +-
.../streams/common/channel/impl/file/FileSink.java | 4 +-
.../common/channel/impl/file/FileSource.java | 2 +-
.../channel/impl/memory/MemoryChannelBuilder.java | 34 +-
.../common/channel/impl/memory/MemorySink.java | 4 +-
.../channel/impl/view/ViewChannelBuilder.java | 51 +--
.../streams/common/channel/impl/view/ViewSink.java | 66 +++
.../common/channel/impl/view/ViewSource.java | 36 +-
.../streams/common/channel/sink/AbstractSink.java | 13 +-
.../channel/sink/AbstractSupportShuffleSink.java | 8 +-
.../sink/AbstractSupportShuffleUDFSink.java | 6 +-
.../common/channel/sink/AbstractUDFSink.java | 23 +-
.../streams/common/channel/sink/ISink.java | 9 +-
.../common/channel/source/AbstractSource.java | 78 ++--
.../streams/common/channel/split/ISplit.java | 5 +-
.../streams/common/component/ComponentCreator.java | 4 +
.../common/configurable/AbstractConfigurable.java | 17 +-
.../common/configurable/BasedConfigurable.java | 9 +
.../common/configurable/IConfigurableService.java | 5 +-
.../streams/common/context/AbstractContext.java | 3 +-
.../streams/common/context/MessageHeader.java | 74 ++--
.../streams/common/datatype/IntDataType.java | 3 +-
.../streams/common/datatype/ShortDataType.java | 2 +-
.../rocketmq/streams/common/model/NameCreator.java | 12 +-
.../NameCreatorContext.java} | 25 +-
.../common/monitor/ConsoleMonitorManager.java | 411 ++++++++++++++++++
.../streams/common/monitor/DataSyncConstants.java | 42 +-
.../streams/common/monitor/HttpClient.java | 116 +++++
.../rocketmq/streams/common/monitor}/HttpUtil.java | 51 ++-
.../rocketmq/streams/common/monitor/IMonitor.java | 16 +
.../monitor/MonitorDataSyncServiceFactory.java | 61 +++
.../common/monitor/TopologyFilterMonitor.java | 16 +
.../common/monitor/group/MonitorCommander.java | 4 +-
.../streams/common/monitor/impl/DipperMonitor.java | 2 +-
.../streams/common/monitor/model/JobStage.java | 350 +++++++++++++++
.../streams/common/monitor/model/TraceIdsDO.java | 126 ++++++
.../common/monitor/model/TraceMonitorDO.java | 250 +++++++++++
.../service/MonitorDataSyncService.java} | 16 +-
.../service/impl/DBMonitorDataSyncImpl.java | 63 +++
.../service/impl/HttpMonitorDataSyncImpl.java | 151 +++++++
.../service/impl/RocketMQMonitorDataSyncImpl.java | 182 ++++++++
.../optimization/IHomologousOptimization.java | 2 +-
.../common/optimization/MessageGlobleTrace.java | 16 +-
.../streams/common/optimization/Re2Engine.java | 47 +-
.../common/optimization/TaskOptimization.java | 17 +-
.../optimization/fingerprint/PreFingerprint.java | 54 ++-
.../AbstractMutilPipelineChainPipline.java | 4 +-
.../streams/common/topology/ChainPipeline.java | 185 ++++----
.../streams/common/topology/ChainStage.java | 21 +-
.../common/topology/builder/PipelineBuilder.java | 25 +-
.../common/topology/model/AbstractStage.java | 135 +++---
.../streams/common/topology/model/Pipeline.java | 41 +-
.../topology/model/PipelineSourceJoiner.java | 48 --
.../common/topology/stages/FilterChainStage.java | 25 +-
.../topology/stages/SubPiplineChainStage.java | 138 ------
.../ViewChainStage.java} | 473 ++++++++++----------
.../common/topology/stages/udf/UDFChainStage.java | 27 +-
.../streams/common/topology/task/StreamsTask.java | 414 +++---------------
.../streams/common/topology/task/TaskAssigner.java | 20 +-
.../streams/common/utils/ConfigurableUtil.java | 6 +-
.../streams/common/utils/DataTypeUtil.java | 6 +-
.../rocketmq/streams/common/utils/FileUtil.java | 46 +-
.../streams/common/utils/NameCreatorUtil.java | 60 ---
.../rocketmq/streams/common/utils/SQLUtil.java | 2 +-
.../rocketmq/streams/common/channel/SinkTest.java | 4 +-
rocketmq-streams-configurable/pom.xml | 11 +-
.../configurable/ConfigurableComponent.java | 17 +-
.../service/AbstractConfigurableService.java | 98 ++---
.../service/impl}/FileConfigureService.java | 6 +-
.../impl}/FileSupportParentConfigureService.java | 2 +-
.../service/impl/HttpConfigureService.java | 377 ++++++++++++++++
.../impl/HttpSupportParentConfigureService.java | 20 +-
.../service/impl}/MemoryConfigureService.java | 2 +-
.../impl}/MemorySupportParentConfigureService.java | 2 +-
rocketmq-streams-connectors/pom.xml | 6 +-
rocketmq-streams-db-operator/pom.xml | 6 +-
.../streams/db/configuable/DBConfigureService.java | 11 +-
rocketmq-streams-dbinit/pom.xml | 6 +-
.../dbinit/mysql/delegate/DBDelegateFactory.java | 5 +
.../dbinit/mysql/delegate/MysqlDelegate.java | 1 +
.../src/main/resources/tables_mysql_innodb.sql | 16 -
rocketmq-streams-dim/pom.xml | 6 +-
rocketmq-streams-examples/pom.xml | 6 +-
.../checkpoint/RemoteCheckpointExample.java | 4 +-
.../mutilconsumer/MultiStreamsExample.java | 4 +-
.../streams/examples/source/FileSourceExample.java | 2 +-
.../examples/source/RocketmqSourceExample4.java | 63 ---
.../src/main/resources/joinData-1.txt | 4 -
.../src/main/resources/joinData-2.txt | 4 -
rocketmq-streams-filter/pom.xml | 5 +-
.../streams/filter/builder/ExpressionBuilder.java | 49 +--
.../filter/engine/impl/DefaultRuleEngine.java | 2 +-
.../expression/ExpressionRelationParser.java | 9 +-
.../expression/ExpressionRelationPaser.java | 107 -----
.../operator/expression/GroupExpression.java | 3 +-
.../dependency/BlinkRuleV2Expression.java | 5 +-
.../optimization/dependency/DependencyTree.java | 10 +-
.../homologous/HomologousOptimization.java | 2 +-
rocketmq-streams-lease/pom.xml | 6 +-
.../log/catalina.out | 0
rocketmq-streams-runner/pom.xml | 22 +-
rocketmq-streams-schedule/pom.xml | 6 +-
rocketmq-streams-script/pom.xml | 6 +-
.../script/function/impl/router/RouteFunction.java | 4 +-
.../streams/script/service/udf/UDFScript.java | 26 +-
.../streams/script/function/FunctionTest.java | 2 +-
rocketmq-streams-serviceloader/pom.xml | 6 +-
rocketmq-streams-state/pom.xml | 6 +-
rocketmq-streams-transport-minio/pom.xml | 6 +-
rocketmq-streams-window/pom.xml | 6 +-
.../rocketmq/streams/window/model/WindowCache.java | 53 ++-
.../window/operator/AbstractShuffleWindow.java | 9 +-
.../window/operator/impl/SessionOperator.java | 36 +-
.../window/shuffle/AbstractSystemChannel.java | 23 +-
.../streams/window/shuffle/ShuffleChannel.java | 31 +-
.../streams/window/trigger/WindowTrigger.java | 10 +-
180 files changed, 5253 insertions(+), 2555 deletions(-)
delete mode 100644 "docs/design/1.RocketMQ-streams\346\225\264\344\275\223\346\236\266\346\236\204.md"
delete mode 100644 "docs/design/2.\346\236\204\345\273\272DataStream.md"
delete mode 100644 "docs/design/3.\345\220\257\345\212\250DataStream.md"
delete mode 100644 "docs/design/4.\346\225\260\346\215\256\347\232\204\346\265\201\350\275\254\350\277\207\347\250\213.md"
delete mode 100644 "docs/design/5.Window\347\256\227\345\255\220\350\247\243\346\236\220.md"
delete mode 100644 "docs/images/Pipeline\347\261\273\345\233\276.png"
create mode 100644 docs/images/img.png
create mode 100644 docs/images/img_1.png
create mode 100644 docs/images/img_2.png
delete mode 100644 docs/images/window.png
delete mode 100644 "docs/images/\346\200\273\344\275\223\346\236\266\346\236\204\345\233\276.png"
delete mode 100644 "docs/images/\346\200\273\344\275\223\350\277\207\347\250\213.png"
delete mode 100644 "docs/images/\346\211\251\345\256\271\345\211\215.png"
delete mode 100644 "docs/images/\346\234\211\347\212\266\346\200\201\347\256\227\345\255\220.png"
delete mode 100644 "docs/images/\347\212\266\346\200\201.png"
delete mode 100644 "docs/images/\347\274\251\345\256\271.png"
copy rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSinkBuilder.java => rocketmq-streams-channel-es/src/main/java/org/apache/rocketmq/streams/es/sink/ESChannelBuilder.java (66%)
create mode 100644 rocketmq-streams-channel-es/src/main/java/org/apache/rocketmq/streams/es/sink/EsClient.java
create mode 100644 rocketmq-streams-channel-kafka/pom.xml
copy rocketmq-streams-channel-syslog/src/main/java/org/apache/rocketmq/streams/syslog/SyslogChannelBuilder.java => rocketmq-streams-channel-kafka/src/main/java/org/apache/rocketmq/streams/kafka/KafkaChannelBuilder.java (67%)
copy rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/memory/MemorySplit.java => rocketmq-streams-channel-kafka/src/main/java/org/apache/rocketmq/streams/kafka/KafkaSplit.java (63%)
create mode 100644 rocketmq-streams-channel-kafka/src/main/java/org/apache/rocketmq/streams/kafka/sink/KafkaSink.java
create mode 100644 rocketmq-streams-channel-kafka/src/main/java/org/apache/rocketmq/streams/kafka/source/KafkaSource.java
create mode 100644 rocketmq-streams-channel-kafka/src/test/java/org/apache/rocketmq/streams/kafka/KafkaChannelTest.java
create mode 100755 rocketmq-streams-channel-kafka/src/test/resources/log4j.xml
create mode 100644 rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ApplicationTest.java
create mode 100644 rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/MqttSourceExample.java
copy rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/{SinkTest.java => ScriptTest.java} (63%)
copy rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/example/{DimTest.java => SplitTest.java} (53%)
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/cache/MappedByteBufferTableWithPrimaryIndex.java
copy rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/{OutputPrintChannel.java => CollectionSink.java} (64%)
copy rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/sink/UDFDefinedSQLParser.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/CollectionSinkBuilder.java (62%)
copy rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/sink/UDFDefinedSQLParser.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/PrintChannelBuilder.java (61%)
copy rocketmq-streams-channel-syslog/src/main/java/org/apache/rocketmq/streams/syslog/SyslogChannelBuilder.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/file/FileChannelBuilder.java (66%)
copy rocketmq-streams-channel-mqtt/src/main/java/org/apache/rocketmq/streams/mqtt/MqttChannelBuilder.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/memory/MemoryChannelBuilder.java (61%)
copy rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSinkBuilder.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/view/ViewChannelBuilder.java (62%)
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/view/ViewSink.java
copy rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/source/UserDefinedSource.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/impl/view/ViewSource.java (53%)
copy rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/{utils/DipperThreadLocalUtil.java => model/NameCreatorContext.java} (58%)
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/ConsoleMonitorManager.java
copy rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/IPullSource.java => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/DataSyncConstants.java (52%)
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/HttpClient.java
copy {rocketmq-streams-channel-http/src/main/java/org/apache/rocketmq/streams/http/source/util => rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor}/HttpUtil.java (87%)
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/MonitorDataSyncServiceFactory.java
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/model/JobStage.java
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/model/TraceIdsDO.java
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/model/TraceMonitorDO.java
copy rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/{optimization/IHomologousOptimization.java => monitor/service/MonitorDataSyncService.java} (63%)
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/service/impl/DBMonitorDataSyncImpl.java
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/service/impl/HttpMonitorDataSyncImpl.java
create mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/monitor/service/impl/RocketMQMonitorDataSyncImpl.java
delete mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/topology/model/PipelineSourceJoiner.java
delete mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/topology/stages/SubPiplineChainStage.java
copy rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/topology/{task/StreamsTask.java => stages/ViewChainStage.java} (52%)
delete mode 100644 rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/NameCreatorUtil.java
rename {rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl}/FileConfigureService.java (95%)
copy {rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl}/FileSupportParentConfigureService.java (96%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/HttpConfigureService.java
rename rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/FileSupportParentConfigureService.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/HttpSupportParentConfigureService.java (71%)
rename {rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl}/MemoryConfigureService.java (98%)
rename {rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl}/MemorySupportParentConfigureService.java (96%)
delete mode 100644 rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/source/RocketmqSourceExample4.java
delete mode 100644 rocketmq-streams-examples/src/main/resources/joinData-1.txt
delete mode 100644 rocketmq-streams-examples/src/main/resources/joinData-2.txt
delete mode 100644 rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/operator/expression/ExpressionRelationPaser.java
rename "docs/design/6.RocketMQ-streams\345\246\202\344\275\225\345\256\236\347\216\260\345\256\271\351\224\231.md" => rocketmq-streams-runner/log/catalina.out (100%)