You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by ka...@apache.org on 2022/06/21 03:54:59 UTC
[rocketmq-streams] branch snapshot-1.0.4 created (now 42abb209)
This is an automated email from the ASF dual-hosted git repository.
karp pushed a change to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
at 42abb209 merge from replaceDB
This branch includes the following new commits:
new 93d7f775 ConfigurableComponent insert and query
new f5a1380f merge 0.1
new 9d3ae58b merge 1.0
new a5a6ddde merge from snapshot-1.0.3
new a5396881 merge from upstream/snapshot-1.0.3
new 840c956f feat(db) remove db dependency besides channel-db
new ef6f7cf6 Merge branch 'replaceDB' into devForEnterprise
new 5e91a37a 1.0.2-preview-SNAPSHOT -> 1.0.3-preview-SNAPSHOT
new ff92d26b fix(example) join example
new f1d507c0 Merge branch 'replaceDB' into devForEnterprise
new 4d94b411 make FileSourceExample runnable
new 5a236975 fix(example) join example
new 0bcaf5bf modify topic
new 24b5bf90 remove mini batch
new aadf7eaa make RocketMQWindowExample runnable
new 42abb209 merge from replaceDB
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[rocketmq-streams] 04/16: merge from snapshot-1.0.3
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit a5a6ddde987adf1143fc5a1f4ed6902dff5368aa
Merge: 93d7f775 9d3ae58b
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 23 15:32:38 2022 +0800
merge from snapshot-1.0.3
NOTICE | 2 +-
README.md | 16 +-
SUMMARY.md | 7 +
docs/README.md | 142 ------
docs/SUMMARY.md | 8 -
...225\264\344\275\223\346\236\266\346\236\204.md" | 33 --
.../2.\346\236\204\345\273\272DataStream.md" | 73 ----
.../3.\345\220\257\345\212\250DataStream.md" | 53 ---
...265\201\350\275\254\350\277\207\347\250\213.md" | 63 ---
...256\227\345\255\220\350\247\243\346\236\220.md" | 55 ---
...256\236\347\216\260\345\256\271\351\224\231.md" | 0
"docs/images/Pipeline\347\261\273\345\233\276.png" | Bin 44207 -> 0 bytes
docs/images/img.png | Bin 0 -> 38684 bytes
docs/images/img_1.png | Bin 0 -> 43711 bytes
docs/images/img_2.png | Bin 0 -> 103151 bytes
docs/images/window.png | Bin 241692 -> 0 bytes
...75\223\346\236\266\346\236\204\345\233\276.png" | Bin 60493 -> 0 bytes
...00\273\344\275\223\350\277\207\347\250\213.png" | Bin 44252 -> 0 bytes
.../\346\211\251\345\256\271\345\211\215.png" | Bin 56733 -> 0 bytes
...12\266\346\200\201\347\256\227\345\255\220.png" | Bin 35766 -> 0 bytes
"docs/images/\347\212\266\346\200\201.png" | Bin 47527 -> 0 bytes
"docs/images/\347\274\251\345\256\271.png" | Bin 51087 -> 0 bytes
docs/quick_start/README.md | 46 --
pom.xml | 118 ++---
quick_start.md | 92 ++--
.../pom.xml | 41 +-
rocketmq-streams-cep/src/test/resources/log4j.xml | 36 ++
rocketmq-streams-channel-db/pom.xml | 4 +-
.../streams/db/sink/AbstractMultiTableSink.java | 12 +-
.../streams/db/sink/DynamicMultipleDBSink.java | 11 +-
.../streams/db/sink/SelfMultiTableSink.java | 2 +-
.../streams/db/sink/SplitBySerialNumber.java | 2 +-
.../streams/db/sink/SplitByTimeMultiTableSink.java | 2 +-
rocketmq-streams-channel-es/pom.xml | 27 +-
.../{ESSinkBuilder.java => ESChannelBuilder.java} | 54 +--
.../rocketmq/streams/es/sink/ESSinkBuilder.java | 1 +
.../streams/es/sink/ESSinkOnlyChannel.java | 43 +-
.../apache/rocketmq/streams/es/sink/EsClient.java | 135 ++++++
rocketmq-streams-channel-http/pom.xml | 4 +-
rocketmq-streams-channel-kafka/pom.xml | 32 ++
.../streams/kafka/KafkaChannelBuilder.java | 52 ++-
.../apache/rocketmq/streams/kafka/KafkaSplit.java | 55 +++
.../rocketmq/streams/kafka/sink/KafkaSink.java | 200 +++++++++
.../rocketmq/streams/kafka/source/KafkaSource.java | 238 ++++++++++
.../rocketmq/streams/kafka/KafkaChannelTest.java | 104 +++++
.../src/test/resources/log4j.xml | 20 +
rocketmq-streams-channel-mqtt/pom.xml | 20 +-
.../rocketmq/streams/mqtt/source/PahoSource.java | 41 +-
rocketmq-streams-channel-rocketmq/pom.xml | 28 +-
.../apache/rocketmq/streams/debug/DebugWriter.java | 92 ++--
.../apache/rocketmq/streams/sink/RocketMQSink.java | 55 ++-
.../rocketmq/streams/RocketMQChannelTest.java | 2 +-
rocketmq-streams-channel-syslog/pom.xml | 19 +-
.../rocketmq/streams/syslog/SyslogChannel.java | 35 +-
.../streams/syslog/SyslogChannelBuilder.java | 4 +
.../streams/syslog/SyslogChannelManager.java | 6 +-
.../rocketmq/streams/syslog/SyslogServer.java | 35 +-
.../rocketmq/streams/syslog/SyslogClient.java | 22 +-
rocketmq-streams-clients/pom.xml | 14 +-
.../streams/client/source/DataStreamSource.java | 19 +
.../streams/client/transform/DataStream.java | 41 +-
.../streams/client/transform/JoinStream.java | 5 +-
.../streams/client/transform/SplitStream.java | 24 +-
.../streams/client/transform/WindowStream.java | 50 ++-
.../rocketmq/streams/client/ApplicationTest.java | 57 +++
.../rocketmq/streams/client/MqttSourceExample.java | 80 ++++
.../{sink/UserDefinedSink.java => ScriptTest.java} | 27 +-
.../apache/rocketmq/streams/client/WindowTest.java | 14 +-
.../rocketmq/streams/client/example/JoinTest.java | 7 +-
.../rocketmq/streams/client/example/SplitTest.java | 31 +-
.../streams/client/sink/UserDefinedSink.java | 2 +-
.../client/sink/UserDefinedSupportShuffleSink.java | 4 +-
rocketmq-streams-commons/pom.xml | 46 +-
.../MappedByteBufferTableWithPrimaryIndex.java | 482 +++++++++++++++++++++
.../streams/common/cache/compress/KVAddress.java | 48 +-
.../streams/common/channel/AbstractChannel.java | 14 +-
.../AbstractSupportShuffleChannelBuilder.java | 2 +-
.../common/channel/builder/IChannelBuilder.java | 25 +-
.../channel/builder/IShuffleChannelBuilder.java | 4 +-
.../common/channel/impl/CollectionSink.java | 39 +-
.../common/channel/impl/CollectionSinkBuilder.java | 34 +-
.../common/channel/impl/PrintChannelBuilder.java | 30 +-
.../channel/impl/file/FileChannelBuilder.java | 45 +-
.../streams/common/channel/impl/file/FileSink.java | 4 +-
.../common/channel/impl/file/FileSource.java | 2 +-
.../channel/impl/memory/MemoryChannelBuilder.java | 64 +++
.../common/channel/impl/memory/MemorySink.java | 4 +-
.../channel/impl/view/ViewChannelBuilder.java | 43 +-
.../streams/common/channel/impl/view/ViewSink.java | 31 +-
.../common/channel/impl/view/ViewSource.java | 66 +++
.../streams/common/channel/sink/AbstractSink.java | 17 +-
.../channel/sink/AbstractSupportShuffleSink.java | 8 +-
.../sink/AbstractSupportShuffleUDFSink.java | 6 +-
.../common/channel/sink/AbstractUDFSink.java | 23 +-
.../streams/common/channel/sink/ISink.java | 9 +-
.../impl/AbstractMultiSplitMessageCache.java | 22 +-
.../common/channel/source/AbstractSource.java | 89 +++-
.../channel/source/AbstractUnreliableSource.java | 2 +-
.../streams/common/channel/split/ISplit.java | 5 +-
.../streams/common/component/ComponentCreator.java | 4 +
.../common/configurable/AbstractConfigurable.java | 17 +-
.../common/configurable/BasedConfigurable.java | 13 +-
.../common/configurable/IConfigurableService.java | 5 +-
.../streams/common/configure/ConfigureFileKey.java | 7 +-
.../streams/common/context/AbstractContext.java | 33 +-
.../streams/common/context/MessageHeader.java | 89 ++--
.../streams/common/datatype/IntDataType.java | 3 +-
.../streams/common/datatype/ShortDataType.java | 2 +-
.../streams/common/interfaces/ISerialize.java | 9 +-
.../rocketmq/streams/common/model/NameCreator.java | 14 +-
.../NameCreatorContext.java} | 36 +-
.../common/monitor/ConsoleMonitorManager.java | 412 ++++++++++++++++++
.../streams/common/monitor/DataSyncConstants.java | 54 +++
.../streams/common/monitor/HttpClient.java | 116 +++++
.../rocketmq/streams/common/monitor/HttpUtil.java | 248 +++++++++++
.../monitor/MonitorDataSyncServiceFactory.java | 61 +++
.../common/monitor/group/MonitorCommander.java | 4 +-
.../streams/common/monitor/impl/DipperMonitor.java | 2 +-
.../streams/common/monitor/model/JobStage.java | 350 +++++++++++++++
.../streams/common/monitor/model/TraceIdsDO.java | 126 ++++++
.../common/monitor/model/TraceMonitorDO.java | 250 +++++++++++
.../service/MonitorDataSyncService.java} | 16 +-
.../service/impl/DBMonitorDataSyncImpl.java | 63 +++
.../service/impl/HttpMonitorDataSyncImpl.java | 151 +++++++
.../service/impl/RocketMQMonitorDataSyncImpl.java | 185 ++++++++
.../optimization/IHomologousOptimization.java | 2 +-
.../common/optimization/MessageGlobleTrace.java | 16 +-
.../streams/common/optimization/Re2Engine.java | 47 +-
.../common/optimization/TaskOptimization.java | 17 +-
.../optimization/fingerprint/FingerprintCache.java | 4 +-
.../optimization/fingerprint/PreFingerprint.java | 54 ++-
.../streams/common/schedule/ScheduleTask.java | 4 +-
.../common/threadpool/ThreadPoolFactory.java | 34 +-
.../AbstractMutilPipelineChainPipline.java | 81 ++--
.../streams/common/topology/ChainPipeline.java | 255 ++++++-----
.../streams/common/topology/ChainStage.java | 32 +-
.../common/topology/builder/PipelineBuilder.java | 139 ++++--
.../common/topology/metric/NotFireReason.java | 176 ++++++++
.../streams/common/topology/metric/StageGroup.java | 248 +++++++++++
.../common/topology/metric/StageMetric.java | 138 ++++++
.../common/topology/model/AbstractStage.java | 177 +++++---
.../streams/common/topology/model/IWindow.java | 3 +-
.../streams/common/topology/model/Pipeline.java | 56 ++-
.../topology/model/PipelineSourceJoiner.java | 48 --
.../topology/shuffle/IShuffleKeyGenerator.java | 11 +-
.../common/topology/shuffle/ShuffleMQCreator.java | 398 ++++++++++-------
.../topology/stages/AbstractWindowStage.java | 5 +-
.../stages/EmptyChainStage.java} | 27 +-
.../common/topology/stages/FilterChainStage.java | 157 ++-----
.../common/topology/stages/JoinChainStage.java | 3 +-
.../common/topology/stages/JoinEndChainStage.java | 11 +-
.../topology/stages/JoinStartChainStage.java | 67 +++
.../common/topology/stages/OutputChainStage.java | 81 ++--
.../topology/stages/ShuffleConsumerChainStage.java | 193 +++++++++
.../topology/stages/ShuffleProducerChainStage.java | 345 +++++++++++++++
.../topology/stages/SubPiplineChainStage.java | 138 ------
.../common/topology/stages/UnionChainStage.java | 3 +-
.../common/topology/stages/UnionEndChainStage.java | 11 +-
...onChainStage.java => UnionStartChainStage.java} | 43 +-
.../ViewChainStage.java} | 479 ++++++++++----------
.../common/topology/stages/udf/UDFChainStage.java | 27 +-
.../streams/common/topology/task/StreamsTask.java | 444 +++----------------
.../streams/common/topology/task/TaskAssigner.java | 20 +-
.../streams/common/utils/ConfigurableUtil.java | 6 +-
.../streams/common/utils/ContantsUtil.java | 33 +-
.../streams/common/utils/DataTypeUtil.java | 16 +-
.../rocketmq/streams/common/utils/FileUtil.java | 89 +++-
.../streams/common/utils/InstantiationUtil.java | 48 +-
.../streams/common/utils/JsonableUtil.java | 5 +
.../rocketmq/streams/common/utils/KryoUtil.java | 214 +++++++++
.../streams/common/utils/NameCreatorUtil.java | 60 ---
.../streams/common/utils/PipelineHTMLUtil.java | 299 +++++++++++++
.../streams/common/utils/PropertiesUtils.java | 2 +-
.../rocketmq/streams/common/utils/ReflectUtil.java | 62 ++-
.../streams/common/utils/SerializeUtil.java | 23 +-
.../streams/common/utils/ServiceLoadUtil.java | 63 +++
.../rocketmq/streams/common/utils/TraceUtil.java | 23 +-
.../rocketmq/streams/common/channel/SinkTest.java | 4 +-
rocketmq-streams-configurable/pom.xml | 9 +-
.../configurable/ConfigurableComponent.java | 17 +-
.../service/AbstractConfigurableService.java | 98 ++---
.../service/impl}/FileConfigureService.java | 2 +-
.../impl}/FileSupportParentConfigureService.java | 2 +-
.../service/impl/HttpConfigureService.java | 377 ++++++++++++++++
.../impl/HttpSupportParentConfigureService.java | 20 +-
.../service/impl}/MemoryConfigureService.java | 2 +-
.../impl}/MemorySupportParentConfigureService.java | 2 +-
rocketmq-streams-connectors/pom.xml | 9 +-
.../{IBounded.java => IScheduleCallback.java} | 9 +-
.../connectors/source/AbstractPullSource.java | 148 ++++---
.../connectors/source/MutilBatchTaskSource.java | 158 +++++++
rocketmq-streams-db-operator/pom.xml | 4 +-
.../streams/db/configuable/DBConfigureService.java | 11 +-
rocketmq-streams-dim/pom.xml | 4 +-
.../intelligence/AbstractIntelligenceCache.java | 5 +-
rocketmq-streams-examples/pom.xml | 20 +-
.../mutilconsumer/MultiStreamsExample.java | 3 +
.../streams/examples/send/ProducerFromFile.java | 8 +-
.../streams/examples/source/FileSourceExample.java | 2 +-
.../src/main/resources/joinData-2.txt | 4 -
rocketmq-streams-filter/pom.xml | 3 +-
.../streams/filter/builder/ExpressionBuilder.java | 49 +--
.../streams/filter/context/RuleContext.java | 31 +-
.../filter/engine/impl/DefaultRuleEngine.java | 29 +-
.../function/expression/CompareFunction.java | 16 +-
.../rocketmq/streams/filter/operator/Rule.java | 58 +++
.../expression/ExpressionRelationParser.java | 9 +-
.../expression/ExpressionRelationPaser.java | 107 -----
.../operator/expression/GroupExpression.java | 3 +-
.../operator/expression/RelationExpression.java | 35 +-
.../PiplineLogFingerprintAnalysis.java | 6 +-
.../dependency/BlinkRuleV2Expression.java | 5 +-
.../optimization/dependency/DependencyTree.java | 22 +-
.../dependency/SimplePipelineTree.java | 3 +-
.../dependency/StateLessDependencyTree.java | 84 ++++
.../optimization/homologous/HomologousCompute.java | 5 +-
.../homologous/HomologousOptimization.java | 13 +-
rocketmq-streams-lease/pom.xml | 4 +-
rocketmq-streams-runner/assembly/distribution.xml | 69 ---
rocketmq-streams-runner/assembly/standalone.xml | 72 ---
rocketmq-streams-runner/bin/start.sh | 58 ---
rocketmq-streams-runner/bin/stop.sh | 33 --
rocketmq-streams-runner/pom.xml | 80 ----
.../src/main/resources/log4j.xml | 51 ---
rocketmq-streams-schedule/pom.xml | 4 +-
.../schedule/job/ConfigurableExecutorJob.java | 30 +-
rocketmq-streams-script/pom.xml | 4 +-
.../function/aggregation/LastValueAccumulator.java | 67 +++
.../function/impl/distinct/DistinctFunction.java | 1 -
.../function/impl/json/JsonCreatorFunction.java | 4 +-
.../function/impl/json/UDTFFieldNameFunction.java | 50 +++
.../script/function/impl/parser/GrokFunction.java | 4 +-
.../function/impl/parser/Paser2JsonFunction.java | 17 +-
.../function/impl/parser/PaserBySplitFunction.java | 44 +-
.../function/impl/parser/RegexParserFunction.java | 24 +-
.../script/function/impl/router/RouteFunction.java | 4 +-
.../script/operator/impl/AggregationScript.java | 19 +-
.../streams/script/service/IAccumulator.java | 4 +-
.../script/service/udf/SimpleUDAFScript.java | 35 +-
.../streams/script/service/udf/UDFScript.java | 26 +-
.../streams/script/function/FunctionTest.java | 19 +-
rocketmq-streams-serviceloader/pom.xml | 4 +-
rocketmq-streams-state/pom.xml | 4 +-
.../streams/state/kv/rocksdb/RocksDBOperator.java | 37 +-
rocketmq-streams-transport-minio/pom.xml | 4 +-
rocketmq-streams-window/pom.xml | 9 +-
.../window/minibatch/MiniBatchMsgCache.java | 58 +++
.../window/minibatch/ShuffleMessageCache.java | 187 ++++++++
.../rocketmq/streams/window/model/WindowCache.java | 182 +++-----
.../streams/window/model/WindowInstance.java | 22 +-
.../window/operator/AbstractShuffleWindow.java | 18 +-
.../streams/window/operator/AbstractWindow.java | 73 +++-
.../window/operator/impl/SessionOperator.java | 2 +-
.../window/operator/impl/WindowOperator.java | 8 +-
.../streams/window/operator/join/JoinWindow.java | 3 +-
.../window/shuffle/AbstractSystemChannel.java | 94 ++--
.../streams/window/shuffle/ShuffleCache.java | 14 +-
.../streams/window/shuffle/ShuffleChannel.java | 54 +--
.../streams/window/state/impl/WindowValue.java | 54 ++-
.../streams/window/trigger/WindowTrigger.java | 1 -
.../rocketmq/streams/window/util/ShuffleUtil.java | 62 +++
.../org/apache/rocketmq/streams/RocksdbTest.java | 2 +-
docs/stream_sink/README.md => stream_sink.md | 19 +-
docs/stream_source/README.md => stream_source.md | 30 +-
.../README.md => stream_transform.md | 0
265 files changed, 9865 insertions(+), 4083 deletions(-)
diff --cc pom.xml
index 70288c26,7caa0cbc..2d139613
--- a/pom.xml
+++ b/pom.xml
@@@ -184,9 -137,9 +136,10 @@@
<exclude>**/*.xml</exclude>
<exclude>**/*.sh</exclude>
<exclude>**/*.out</exclude>
+ <exclude>**/*.sql</exclude>
<exclude>**/*.properties</exclude>
<exclude>docs/**/*</exclude>
+ <exclude>**/*.sql</exclude>
</excludes>
</configuration>
</plugin>
diff --cc rocketmq-streams-channel-es/pom.xml
index 15723b3c,fe50aeef..b4ce4d85
--- a/rocketmq-streams-channel-es/pom.xml
+++ b/rocketmq-streams-channel-es/pom.xml
@@@ -20,8 -7,8 +7,8 @@@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-SNAPSHOT</version>
+ <version>1.0.2-preview-SNAPSHOT</version>
- </parent>
+ </parent>
<artifactId>rocketmq-streams-channel-es</artifactId>
<name>ROCKETMQ STREAMS :: channel-es</name>
<packaging>jar</packaging>
diff --cc rocketmq-streams-channel-kafka/pom.xml
index 00000000,89bd645c..76b73909
mode 000000,100644..100644
--- a/rocketmq-streams-channel-kafka/pom.xml
+++ b/rocketmq-streams-channel-kafka/pom.xml
@@@ -1,0 -1,32 +1,32 @@@
+ <?xml version="1.0" encoding="UTF-8"?>
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>rocketmq-streams</artifactId>
+ <groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-SNAPSHOT</version>
++ <version>1.0.2-preview-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>rocketmq-streams-channel-kafka</artifactId>
+ <name>ROCKETMQ STREAMS :: channel-kafka</name>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.12</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-commons</artifactId>
+ </dependency>
+ </dependencies>
+ </project>
diff --cc rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java
index bb6e9454,d825603a..1de92e31
--- a/rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java
+++ b/rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java
@@@ -17,8 -17,10 +17,11 @@@
package org.apache.rocketmq.streams.sink;
+ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
+ import java.util.Comparator;
+ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@@ -27,8 -29,10 +30,9 @@@ import java.util.Set
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
-import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
+ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
@@@ -149,10 -154,10 +154,6 @@@ public class RocketMQSink extends Abstr
destroy();
producer = new DefaultMQProducer(groupName + "producer", true, null);
try {
-- //please not use the code，the name srv addr may be empty in jmenv
--// if (this.namesrvAddr == null || "".equals(this.namesrvAddr)) {
--// throw new RuntimeException("namesrvAddr can not be null.");
--// }
if (StringUtil.isNotEmpty(this.namesrvAddr)) {
producer.setNamesrvAddr(this.namesrvAddr);
@@@ -263,7 -275,7 +272,7 @@@
@Override
public int getSplitNum() {
- List<ISplit> splits = getSplitList();
- List<ISplit<?, ?>> splits = getSplitList();
++ List<ISplit<?,?>> splits = getSplitList();
if (splits == null || splits.size() == 0) {
return 0;
}
diff --cc rocketmq-streams-clients/pom.xml
index 8fb31672,94c7f3e7..2d15efce
--- a/rocketmq-streams-clients/pom.xml
+++ b/rocketmq-streams-clients/pom.xml
@@@ -50,6 -57,18 +57,11 @@@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-filter</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-syslog</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-channel-kafka</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-dbinit</artifactId>
- </dependency>
++
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-window</artifactId>
diff --cc rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
index 9422519a,5004f230..1e4b15b6
--- a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
@@@ -178,6 -179,25 +178,25 @@@ public class DataStreamSource
return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
}
- public DataStream fromKafka(String endpoint, String topic, String groupName) {
- return fromKafka(endpoint, topic, groupName, true);
- }
-
- public DataStream fromKafka(String endpoint, String topic, String groupName, Boolean isJson) {
- return fromKafka(endpoint, topic, groupName, isJson, 1);
- }
-
- public DataStream fromKafka(String endpoint, String topic, String groupName, Boolean isJson, int maxThread) {
- KafkaSource kafkaChannel = new KafkaSource();
- kafkaChannel.setBootstrapServers(endpoint);
- kafkaChannel.setTopic(topic);
- kafkaChannel.setGroupName(groupName);
- kafkaChannel.setJsonData(isJson);
- kafkaChannel.setMaxThread(maxThread);
- this.mainPipelineBuilder.setSource(kafkaChannel);
- return new DataStream(this.mainPipelineBuilder, null);
- }
++// public DataStream fromKafka(String endpoint, String topic, String groupName) {
++// return fromKafka(endpoint, topic, groupName, true);
++// }
++//
++// public DataStream fromKafka(String endpoint, String topic, String groupName, Boolean isJson) {
++// return fromKafka(endpoint, topic, groupName, isJson, 1);
++// }
++//
++// public DataStream fromKafka(String endpoint, String topic, String groupName, Boolean isJson, int maxThread) {
++// KafkaSource kafkaChannel = new KafkaSource();
++// kafkaChannel.setBootstrapServers(endpoint);
++// kafkaChannel.setTopic(topic);
++// kafkaChannel.setGroupName(groupName);
++// kafkaChannel.setJsonData(isJson);
++// kafkaChannel.setMaxThread(maxThread);
++// this.mainPipelineBuilder.setSource(kafkaChannel);
++// return new DataStream(this.mainPipelineBuilder, null);
++// }
+
public DataStream from(ISource<?> source) {
this.mainPipelineBuilder.setSource(source);
return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
diff --cc rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
index 80ae510e,96e24a0a..563af478
--- a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
@@@ -51,8 -51,11 +51,10 @@@ import org.apache.rocketmq.streams.comm
import org.apache.rocketmq.streams.common.topology.ChainStage;
import org.apache.rocketmq.streams.common.topology.builder.IStageBuilder;
import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
-import org.apache.rocketmq.streams.common.topology.model.AbstractRule;
import org.apache.rocketmq.streams.common.topology.model.Union;
import org.apache.rocketmq.streams.common.topology.stages.FilterChainStage;
+ import org.apache.rocketmq.streams.common.topology.stages.ShuffleConsumerChainStage;
+ import org.apache.rocketmq.streams.common.topology.stages.ShuffleProducerChainStage;
import org.apache.rocketmq.streams.common.topology.stages.udf.StageBuilder;
import org.apache.rocketmq.streams.common.topology.stages.udf.UDFChainStage;
import org.apache.rocketmq.streams.common.topology.stages.udf.UDFUnionChainStage;
@@@ -646,6 -669,13 +667,13 @@@ public class DataStream implements Seri
return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
}
- public DataStream toKafka(String bootstrapServers, String topic) {
- KafkaSink kafkaSink = new KafkaSink(bootstrapServers, topic);
- ChainStage<?> output = this.mainPipelineBuilder.createStage(kafkaSink);
- this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
- return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
- }
++// public DataStream toKafka(String bootstrapServers, String topic) {
++// KafkaSink kafkaSink = new KafkaSink(bootstrapServers, topic);
++// ChainStage<?> output = this.mainPipelineBuilder.createStage(kafkaSink);
++// this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
++// return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
++// }
+
public DataStream toEnhanceDBSink(String url, String userName, String password, String tableName) {
EnhanceDBSink sink = new EnhanceDBSink(url, userName, password, tableName);
ChainStage<?> output = this.mainPipelineBuilder.createStage(sink);
diff --cc rocketmq-streams-commons/pom.xml
index 5ea155d4,72af956e..2e73c6f8
--- a/rocketmq-streams-commons/pom.xml
+++ b/rocketmq-streams-commons/pom.xml
@@@ -98,11 -101,36 +101,40 @@@
<artifactId>re2j</artifactId>
<version>1.6</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>de.ruedigermoeller</groupId>
+ <artifactId>fst</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ <version>5.3.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.13</version>
+ </dependency>
</dependencies>
</project>
diff --cc rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/source/AbstractSource.java
index 939b1d54,320cbd12..2bc7e55f
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/source/AbstractSource.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/channel/source/AbstractSource.java
@@@ -18,9 -18,10 +18,11 @@@ package org.apache.rocketmq.streams.com
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
+
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@@ -58,15 -58,22 +60,23 @@@ import org.apache.rocketmq.streams.comm
public abstract class AbstractSource extends BasedConfigurable implements ISource<AbstractSource>, ILifeCycle {
public static String CHARSET = "UTF-8";
+ /**
+ * 输入的消息是否为json
+ */
+ protected Boolean isJsonData = true;
+ /**
+ * 输入的消息是否为json array
+ */
+ protected Boolean msgIsJsonArray = false;
- protected Boolean isJsonData = true;//输入的消息是否为json
- protected Boolean msgIsJsonArray = false;//输入的消息是否为json array
@ENVDependence
- protected String groupName;//group name
+ protected String groupName;
+
protected int maxThread = Runtime.getRuntime().availableProcessors();
+
@ENVDependence
protected String topic = "";
+ protected String namesrvAddr;
/**
* 多长时间做一次checkpoint
*/
@@@ -136,11 -167,9 +170,10 @@@
* @param message
* @return
*/
- public AbstractContext doReceiveMessage(JSONObject message, boolean needSetCheckPoint, String queueId, String offset) {
+ public AbstractContext doReceiveMessage(JSONObject message, boolean needSetCheckPoint, String queueId,
+ String offset) {
Message msg = createMessage(message, queueId, offset, needSetCheckPoint);
- AbstractContext context = executeMessage(msg);
- return context;
+ return executeMessage(msg);
}
/**
diff --cc rocketmq-streams-connectors/pom.xml
index 8c8277e3,d544e3d5..e557a19a
mode 100755,100644..100644
--- a/rocketmq-streams-connectors/pom.xml
+++ b/rocketmq-streams-connectors/pom.xml
diff --cc rocketmq-streams-examples/pom.xml
index 62a4038c,da6a130b..7ced0cb0
--- a/rocketmq-streams-examples/pom.xml
+++ b/rocketmq-streams-examples/pom.xml
@@@ -39,6 -41,6 +41,22 @@@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-clients</artifactId>
</dependency>
++ <dependency>
++ <groupId>com.alibaba</groupId>
++ <artifactId>fastjson</artifactId>
++ </dependency>
++ <dependency>
++ <groupId>junit</groupId>
++ <artifactId>junit</artifactId>
++ </dependency>
++ <dependency>
++ <groupId>org.apache.rocketmq</groupId>
++ <artifactId>rocketmq-client</artifactId>
++ </dependency>
++ <dependency>
++ <groupId>org.apache.rocketmq</groupId>
++ <artifactId>rocketmq-streams-commons</artifactId>
++ </dependency>
</dependencies>
<packaging>jar</packaging>
diff --cc rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MultiStreamsExample.java
index f5f3d46c,2b823e44..8835e943
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MultiStreamsExample.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/mutilconsumer/MultiStreamsExample.java
@@@ -24,13 -23,11 +24,15 @@@ import com.alibaba.fastjson.JSONObject
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+
++import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.streams.client.StreamBuilder;
import org.apache.rocketmq.streams.client.source.DataStreamSource;
import org.apache.rocketmq.streams.client.strategy.WindowStrategy;
++import org.apache.rocketmq.streams.client.transform.DataStream;
import org.apache.rocketmq.streams.client.transform.window.Time;
import org.apache.rocketmq.streams.client.transform.window.TumblingWindow;
+import org.apache.rocketmq.streams.examples.send.ProducerFromFile;
import static org.apache.rocketmq.streams.examples.aggregate.Constant.NAMESRV_ADDRESS;
import static org.apache.rocketmq.streams.examples.aggregate.Constant.RMQ_CONSUMER_GROUP_NAME;
@@@ -64,6 -61,6 +66,7 @@@ public class MultiStreamsExample
private static void runOneStreamsClient(int index) {
DataStreamSource source = StreamBuilder.dataStream("namespace" + index, "pipeline" + index);
++
source.fromRocketmq(
RMQ_TOPIC,
RMQ_CONSUMER_GROUP_NAME,
diff --cc rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/send/ProducerFromFile.java
index 163d8116,58d3710c..0508c303
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/send/ProducerFromFile.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/send/ProducerFromFile.java
@@@ -17,8 -17,8 +17,12 @@@
*
*/
-package org.apache.rocketmq.streams.examples.aggregate;
+package org.apache.rocketmq.streams.examples.send;
+
++import org.apache.rocketmq.client.producer.DefaultMQProducer;
++import org.apache.rocketmq.common.message.Message;
++import org.apache.rocketmq.remoting.common.RemotingHelper;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
@@@ -26,44 -26,18 +30,40 @@@ import java.io.IOException
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.client.producer.DefaultMQProducer;
-import org.apache.rocketmq.client.producer.SendResult;
-import org.apache.rocketmq.common.message.Message;
-import org.apache.rocketmq.remoting.common.RemotingHelper;
+import java.util.concurrent.atomic.AtomicLong;
- import org.apache.rocketmq.client.producer.DefaultMQProducer;
- import org.apache.rocketmq.common.message.Message;
- import org.apache.rocketmq.remoting.common.RemotingHelper;
-
public class ProducerFromFile {
+ private static final DefaultMQProducer producer = new DefaultMQProducer("test-group");
+ private static final AtomicLong count = new AtomicLong(0);
+ private static boolean init = false;
- public static void produce(String filePath, String nameServ, String topic) {
- try {
- DefaultMQProducer producer = new DefaultMQProducer("test-group");
+ private static synchronized void initProducer(String nameServ) throws Throwable {
+ if (!init) {
producer.setNamesrvAddr(nameServ);
producer.start();
+ init = true;
+ }
+ }
+
+ public static void produceInLoop(String filePath, String nameServ, String topic, long interval) {
+ while (true) {
+ try {
+ produce(filePath, nameServ, topic, false);
+
+ Thread.sleep(interval);
+
+ if (count.get() % 500 == 0) {
+ System.out.println("send message num: " + count.get());
+ }
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+ }
+ }
+
+ public static void produce(String filePath, String nameServ, String topic, boolean shutdown) {
+ try {
+ initProducer(nameServ);
List<String> result = ProducerFromFile.read(filePath);
diff --cc rocketmq-streams-window/pom.xml
index 40fcbf94,7a2be2dd..3ce4dac8
--- a/rocketmq-streams-window/pom.xml
+++ b/rocketmq-streams-window/pom.xml
@@@ -48,10 -50,6 +50,13 @@@
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
</dependency>
-
-
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-tools</artifactId>
+ </dependency>
++ <dependency>
++ <groupId>org.apache.rocketmq</groupId>
++ <artifactId>rocketmq-streams-commons</artifactId>
++ </dependency>
</dependencies>
</project>
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/minibatch/MiniBatchMsgCache.java
index 00000000,39629cc1..a36309e2
mode 000000,100644..100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/minibatch/MiniBatchMsgCache.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/minibatch/MiniBatchMsgCache.java
@@@ -1,0 -1,76 +1,58 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.rocketmq.streams.window.minibatch;
+
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+ import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.rocketmq.streams.common.channel.sinkcache.IMessageFlushCallBack;
+ import org.apache.rocketmq.streams.common.channel.sinkcache.impl.AbstractMultiSplitMessageCache;
+ import org.apache.rocketmq.streams.common.channel.sinkcache.impl.MessageCache;
+ import org.apache.rocketmq.streams.common.channel.split.ISplit;
+ import org.apache.rocketmq.streams.common.context.IMessage;
-import org.apache.rocketmq.streams.common.context.Message;
-import org.apache.rocketmq.streams.common.context.MessageHeader;
+ import org.apache.rocketmq.streams.common.topology.shuffle.IShuffleKeyGenerator;
-import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
-import org.apache.rocketmq.streams.common.utils.ReflectUtil;
-import org.apache.rocketmq.streams.common.utils.StringUtil;
-import org.apache.rocketmq.streams.script.operator.impl.AggregationScript;
-import org.apache.rocketmq.streams.window.model.WindowInstance;
+ import org.apache.rocketmq.streams.window.operator.AbstractShuffleWindow;
-import org.apache.rocketmq.streams.window.operator.AbstractWindow;
-import org.apache.rocketmq.streams.window.state.impl.WindowValue;
-import org.apache.rocketmq.streams.window.util.ShuffleUtil;
+
+ public class MiniBatchMsgCache extends AbstractMultiSplitMessageCache<Pair<ISplit,IMessage>> {
+ public static String SHUFFLE_KEY="shuffle_key";
+
+
+
+ protected transient IShuffleKeyGenerator shuffleKeyGenerator;
+ protected transient AbstractShuffleWindow window;
+
+
+
+
+ public MiniBatchMsgCache(
+ IMessageFlushCallBack<Pair<ISplit,IMessage>> flushCallBack, IShuffleKeyGenerator shuffleKeyGenerator,
+ AbstractShuffleWindow window) {
+ super(flushCallBack);
+ this.shuffleKeyGenerator=shuffleKeyGenerator;
+ this.window=window;
+ }
+
+
+ @Override protected String createSplitId(Pair<ISplit, IMessage> msg) {
+ return msg.getLeft().getQueueId();
+ }
+
+ @Override protected MessageCache createMessageCache() {
+ ShuffleMessageCache messageCache=new ShuffleMessageCache(this.flushCallBack);
+ messageCache.setWindow(window);
+ messageCache.setShuffleKeyGenerator(shuffleKeyGenerator);
+ return messageCache;
+ }
+ }
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowInstance.java
index c749bd84,fed18863..4062b1f5
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowInstance.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowInstance.java
@@@ -116,6 -122,19 +116,10 @@@ public class WindowInstance extends Ent
return windowInstance;
}
- /**
- * 创建window instance的唯一ID
- *
- * @return
- */
+ public String createWindowInstanceId() {
+ return MapKeyUtil.createKey(splitId, windowNameSpace, windowName, windowInstanceName, startTime, endTime);
+ }
+
- public String createWindowInstanceIdWithoutSplitid() {
- return MapKeyUtil.createKey(windowNameSpace, windowName, windowInstanceName, startTime, endTime);
- }
-
public String createWindowInstanceTriggerId() {
return MapKeyUtil.createKey(splitId, windowNameSpace, windowName, windowInstanceName, startTime, endTime, fireTime);
}
@@@ -162,7 -254,20 +165,18 @@@
* @return
* @Param isWindowInstance2DB 如果是秒级窗口，可能windowinstacne不必存表，只在内存保存，可以通过这个标志设置
*/
- public static List<WindowInstance> getOrCreateWindowInstance(AbstractWindow window, Long occurTime,
- int timeUnitAdjust, String queueId) {
+ public static List<WindowInstance> getOrCreateWindowInstance(AbstractWindow window, Long occurTime, int timeUnitAdjust, String queueId) {
+ return getOrCreateWindowInstance(window,occurTime,timeUnitAdjust,queueId,false);
+ }
+ /**
+ * 查询或者创建Window的实例，滑动窗口有可能返回多个，滚动窗口返回一个
+ *
+ * @param window
+ * @param occurTime
+ * @return
+ * @Param isWindowInstance2DB 如果是秒级窗口，可能windowinstacne不必存表，只在内存保存，可以通过这个标志设置
+ */
- public static List<WindowInstance> getOrCreateWindowInstance(AbstractWindow window, Long occurTime,
- int timeUnitAdjust, String queueId, boolean isCreateOnly) {
++ public static List<WindowInstance> getOrCreateWindowInstance(AbstractWindow window, Long occurTime, int timeUnitAdjust, String queueId, boolean isCreateOnly) {
int windowSlideInterval = window.getSlideInterval();
int windowSizeInterval = window.getSizeInterval();
if (windowSlideInterval == 0) {
@@@ -246,10 -345,9 +264,10 @@@
}
}
List<WindowInstance> lostInstanceList = null;
+ //todo 这里针对lost的都创建一次
lostInstanceList = WindowInstance.createWindowInstances(window, lostWindowTimeList, lostFireList, queueId);
instanceList.addAll(lostInstanceList);
- if (CollectionUtil.isNotEmpty(lostInstanceList)) {
+ if (CollectionUtil.isNotEmpty(lostInstanceList)&&!isCreateOnly) {
for (WindowInstance windowInstance : instanceList) {
List<WindowInstance> emitInstances = createEmitWindowInstance(window, windowInstance);
if (emitInstances != null && emitInstances.size() > 0) {
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
index 070a40f0,5338734f..7c78f478
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
@@@ -85,8 -74,7 +100,7 @@@ public abstract class AbstractShuffleWi
Set<String> splitIds = new HashSet<>();
splitIds.add(windowInstance.getSplitId());
shuffleChannel.flush(splitIds);
-
- return fireWindowInstance(windowInstance, windowInstance.getSplitId(), queueId2Offset);
+ return doFireWindowInstance(windowInstance);
}
/**
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractWindow.java
index 49d79c5b,8fd07737..eb548aa6
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractWindow.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractWindow.java
@@@ -17,6 -17,17 +17,8 @@@
package org.apache.rocketmq.streams.window.operator;
import com.alibaba.fastjson.JSONObject;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
++
+ import javafx.util.Pair;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@@ -137,8 -164,8 +145,9 @@@ public abstract class AbstractWindow ex
protected boolean isLocalStorageOnly = true;//是否只用本地存储，可以提高性能，但不保证可靠性
protected String reduceSerializeValue;//用户自定义的operator的序列化字节数组，做了base64解码
protected transient IReducer reducer;
-
+ protected transient Long maxPartitionNum = 100000000L;
+ protected String mapFunctionSerializeValue;//用户自定义的operator的序列化字节数组，做了base64解码
+ protected transient MapFunction<JSONObject, Pair<WindowInstance, JSONObject>> mapFunction;
/**
* the computed column and it's process of computing
*/
@@@ -164,12 -191,23 +173,13 @@@
*/
protected transient String WINDOW_NAME;
- /**
- * 内部使用,定期检查窗口有没有触发
- */
- //protected transient ScheduledExecutorService fireWindowInstanceChecker =new ScheduledThreadPoolExecutor(3);
-
- // protected transient ExecutorService deleteService = Executors.newSingleThreadExecutor();
protected volatile transient WindowCache windowCache;
- protected transient WindowStorage storage;
+ protected transient IStorage storage;
protected transient WindowTrigger windowFireSource;
- protected transient SQLCache sqlCache;
protected transient EventTimeManager eventTimeManager;
+ protected transient ISink contextMsgSink;
- //create and save window instacne max partitionNum and window max eventTime
- protected transient IWindowMaxValueManager windowMaxValueManager;
-
public AbstractWindow() {
setType(IWindow.TYPE);
}
@@@ -199,8 -242,12 +209,12 @@@
byte[] bytes = Base64Utils.decode(this.reduceSerializeValue);
reducer = InstantiationUtil.deserializeObject(bytes);
}
+ if (StringUtil.isNotEmpty(this.mapFunctionSerializeValue)) {
+ byte[] bytes = Base64Utils.decode(this.mapFunctionSerializeValue);
+ this.mapFunction = InstantiationUtil.deserializeObject(bytes);
+ }
eventTimeManager = new EventTimeManager();
- windowMaxValueManager = new WindowMaxValueManager(this, sqlCache);
+
return success;
}
@@@ -306,11 -347,11 +320,12 @@@
* @param message
* @return
*/
- protected String generateShuffleKey(IMessage message) {
+ @Override
+ public String generateShuffleKey(IMessage message) {
if (StringUtil.isEmpty(groupByFieldName)) {
- return null;
+ return "globle_window";
}
+
JSONObject msg = message.getMessageBody();
String[] fieldNames = groupByFieldName.split(";");
String[] values = new String[fieldNames.length];
@@@ -773,13 -845,45 +796,53 @@@
this.maxDelay = maxDelay;
}
+ public Long getMaxPartitionNum() {
+ return maxPartitionNum;
+ }
+
+ public void setMaxPartitionNum(Long maxPartitionNum) {
+ this.maxPartitionNum = maxPartitionNum;
+ }
+
public abstract boolean supportBatchMsgFinish();
+
+ public ISink getContextMsgSink() {
+ return contextMsgSink;
+ }
+
+ public String getContextMsgSinkName() {
+ return contextMsgSinkName;
+ }
+
+ public void setContextMsgSinkName(String contextMsgSinkName) {
+ this.contextMsgSinkName = contextMsgSinkName;
+ }
+
+ public String getMapFunctionSerializeValue() {
+ return mapFunctionSerializeValue;
+ }
+
+ public void setMapFunctionSerializeValue(String mapFunctionSerializeValue) {
+ this.mapFunctionSerializeValue = mapFunctionSerializeValue;
+ }
+
+ public void saveMsgContext(String queueId,WindowInstance windowInstance, List<IMessage> messages) {
+ if(this.mapFunction!=null&&this.contextMsgSink!=null){
+ if(messages!=null){
+ for(IMessage message:messages){
+ JSONObject msg=message.getMessageBody();
+ try {
+ msg=this.mapFunction.map(new Pair(windowInstance,msg));
+ Message copyMsg=new Message(msg);
+ copyMsg.getHeader().setQueueId(queueId);
+ copyMsg.getHeader().setOffset(message.getHeader().getOffset());
+ this.contextMsgSink.batchAdd(copyMsg);
+ } catch (Exception e) {
+ throw new RuntimeException("save window context msg error ",e);
+ }
+ }
+ this.contextMsgSink.flush();
+ }
+ }
+ }
}
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java
index 65ac261b,8da03987..0c10a9bf
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java
@@@ -515,13 -481,12 +515,13 @@@ public class SessionOperator extends Wi
store(lastValueMap, instance, queueId);
}
+
@Override
public long incrementAndGetSplitNumber(WindowInstance instance, String shuffleId) {
- long number = super.incrementAndGetSplitNumber(instance, shuffleId);
- if (number > 900000000) {
- this.getWindowMaxValueManager().resetSplitNum(instance, shuffleId);
+ long numer = super.incrementAndGetSplitNumber(instance, shuffleId);
+ if (numer > 900000000) {
+ this.storage.putMaxPartitionNum(shuffleId, instance.getWindowInstanceId(), numer);
}
- return number;
+ return numer;
}
- }
+ }
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/WindowOperator.java
index 65f326f6,d2806081..de6dce22
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/WindowOperator.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/WindowOperator.java
@@@ -16,113 -16,155 +16,119 @@@
*/
package org.apache.rocketmq.streams.window.operator.impl;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.rocketmq.streams.common.channel.split.ISplit;
++
import org.apache.rocketmq.streams.common.context.IMessage;
import org.apache.rocketmq.streams.common.context.MessageOffset;
-import org.apache.rocketmq.streams.common.utils.CollectionUtil;
-import org.apache.rocketmq.streams.common.utils.DateUtil;
import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
import org.apache.rocketmq.streams.common.utils.StringUtil;
-import org.apache.rocketmq.streams.db.driver.batchloader.IRowOperator;
-import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
-import org.apache.rocketmq.streams.script.operator.impl.AggregationScript;
import org.apache.rocketmq.streams.window.debug.DebugWriter;
-import org.apache.rocketmq.streams.window.model.FunctionExecutor;
import org.apache.rocketmq.streams.window.model.WindowInstance;
import org.apache.rocketmq.streams.window.operator.AbstractShuffleWindow;
-import org.apache.rocketmq.streams.window.operator.AbstractWindow;
-import org.apache.rocketmq.streams.window.sqlcache.impl.FiredNotifySQLElement;
import org.apache.rocketmq.streams.window.state.WindowBaseValue;
import org.apache.rocketmq.streams.window.state.impl.WindowValue;
-import org.apache.rocketmq.streams.window.storage.IWindowStorage;
-import org.apache.rocketmq.streams.window.storage.ShufflePartitionManager;
-import org.apache.rocketmq.streams.window.storage.WindowStorage.WindowBaseValueIterator;
-
-public class WindowOperator extends AbstractShuffleWindow {
+import org.apache.rocketmq.streams.window.storage.IteratorWrap;
+import org.apache.rocketmq.streams.window.storage.RocksdbIterator;
+import org.apache.rocketmq.streams.window.storage.WindowType;
- import java.util.*;
- private static final String ORDER_BY_SPLIT_NUM = "_order_by_split_num_";//key=_order;queueid,windowinstanceid,partitionNum
++import java.util.ArrayList;
++import java.util.Comparator;
++import java.util.HashMap;
++import java.util.List;
++import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+public class WindowOperator extends AbstractShuffleWindow {
public WindowOperator() {
super();
}
- protected transient boolean supportQuickStoreModel=false;
- protected transient List<String> schema=new ArrayList<>();
+ @Override
+ public int doFireWindowInstance(WindowInstance instance) {
+ String windowInstanceId = instance.getWindowInstanceId();
+ String queueId = instance.getSplitId();
- @Deprecated
- public WindowOperator(String timeFieldName, int windowPeriodMinute) {
- super();
- super.timeFieldName = timeFieldName;
- super.sizeInterval = windowPeriodMinute;
- }
+ RocksdbIterator<WindowBaseValue> rocksdbIterator = storage.getWindowBaseValue(queueId, windowInstanceId, WindowType.NORMAL_WINDOW, null);
- @Deprecated
- public WindowOperator(String timeFieldName, int windowPeriodMinute, String calFieldName) {
- super();
- super.timeFieldName = timeFieldName;
- super.sizeInterval = windowPeriodMinute;
- }
+ ArrayList<WindowValue> windowValues = new ArrayList<>();
+ while (rocksdbIterator.hasNext()) {
+ IteratorWrap<WindowBaseValue> next = rocksdbIterator.next();
+ WindowValue data = (WindowValue)next.getData();
+ windowValues.add(data);
+ }
+
+ windowValues.sort(Comparator.comparingLong(WindowBaseValue::getPartitionNum));
- public WindowOperator(int sizeInterval, String groupByFieldName, Map<String, String> select) {
- this.sizeInterval = sizeInterval;
- this.slideInterval = sizeInterval;
- this.groupByFieldName = groupByFieldName;
- this.setSelectMap(select);
+ int fireCount = sendBatch(windowValues, queueId, 0);
+
+ clearFire(instance);
+
+ return fireCount;
}
- protected transient AtomicInteger shuffleCount = new AtomicInteger(0);
- protected transient AtomicInteger fireCountAccumulator = new AtomicInteger(0);
- protected transient AtomicLong fireCost=new AtomicLong(0);
- @Override
- public int fireWindowInstance(WindowInstance instance, String queueId, Map<String, String> queueId2Offset) {
- List<WindowValue> windowValues = new ArrayList<>();
- int fireCount = 0;
- //long startTime = System.currentTimeMillis();
- //int sendCost = 0;
- // int currentCount = 0;
- //for(String queueId:currentQueueIds){
- WindowBaseValueIterator<WindowBaseValue> it = storage.loadWindowInstanceSplitData(getOrderBypPrefix(), queueId, instance.createWindowInstanceId(), null, getWindowBaseValueClass());
- if (queueId2Offset != null) {
- String offset = queueId2Offset.get(queueId);
- if (StringUtil.isNotEmpty(offset)) {
- it.setPartitionNum(Long.valueOf(offset));
- }
+ private int sendBatch(List<WindowValue> windowValues, String queueId, int fireCount) {
+ if (windowValues == null || windowValues.size() == 0) {
+ return fireCount;
}
- while (it.hasNext()) {
- WindowBaseValue windowBaseValue = it.next();
- if (windowBaseValue == null) {
- continue;
- }
- WindowValue windowValue = (WindowValue) windowBaseValue;
-
- Integer currentValue = getValue(windowValue, "total");
-
- fireCountAccumulator.addAndGet(currentValue);
- windowValues.add((WindowValue) windowBaseValue);
- if (windowValues.size() >= windowCache.getBatchSize()) {
- long sendFireCost = System.currentTimeMillis();
- sendFireMessage(windowValues, queueId);
- //sendCost += (System.currentTimeMillis() - sendFireCost);
- fireCount += windowValues.size();
- windowValues = new ArrayList<>();
- }
- }
- if (windowValues.size() > 0) {
- long sendFireCost = System.currentTimeMillis();
+ if (windowValues.size() <= windowCache.getBatchSize()) {
sendFireMessage(windowValues, queueId);
- // sendCost += (System.currentTimeMillis() - sendFireCost);
+
fireCount += windowValues.size();
+
+ return fireCount;
+ } else {
+ ArrayList<WindowValue> temp = new ArrayList<>();
+ for (int i = 0; i < windowCache.getBatchSize(); i++) {
+ temp.add(windowValues.remove(i));
+ }
+
+ sendFireMessage(temp, queueId);
+
+ return sendBatch(windowValues, queueId, fireCount + windowCache.getBatchSize());
}
- clearFire(instance);
- this.sqlCache.addCache(new FiredNotifySQLElement(queueId, instance.createWindowInstanceId()));
- //long cost= this.fireCost.addAndGet(System.currentTimeMillis()-startTime);
- // System.out.println("fire cost is "+cost+" "+ DateUtil.getCurrentTimeString());
- return fireCount;
}
- protected transient Map<String, Integer> shuffleWindowInstanceId2MsgCount = new HashMap<>();
- protected transient int windowvaluecount = 0;
- protected transient AtomicLong shuffleCost=new AtomicLong(0);
+
@Override
public void shuffleCalculate(List<IMessage> messages, WindowInstance instance, String queueId) {
+ Long startTime=System.currentTimeMillis();
DebugWriter.getDebugWriter(getConfigureName()).writeShuffleCalcultateReceveMessage(instance, messages, queueId);
+
List<String> sortKeys = new ArrayList<>();
Map<String, List<IMessage>> groupBy = groupByGroupName(messages, sortKeys);
- Set<String> groupByKeys = groupBy.keySet();
- List<String> storeKeys = new ArrayList<>();
- for (String groupByKey : groupByKeys) {
- String storeKey = createStoreKey(queueId, groupByKey, instance);
- storeKeys.add(storeKey);
+
+ RocksdbIterator<WindowBaseValue> windowBaseValue = storage.getWindowBaseValue(queueId, instance.getWindowInstanceId(), WindowType.NORMAL_WINDOW, null);
+
+ ArrayList<WindowBaseValue> windowValues = new ArrayList<>();
+ while (windowBaseValue.hasNext()) {
+ IteratorWrap<WindowBaseValue> next = windowBaseValue.next();
+ windowValues.add(next.getData());
}
- Map<String, WindowBaseValue> allWindowValues = new HashMap<>();
- //从存储中，查找window value对象，value是对象的json格式
- Map<String, WindowBaseValue> existWindowValues = storage.multiGet(getWindowBaseValueClass(), storeKeys, instance.createWindowInstanceId(), queueId);
- // Iterator<Entry<String, List<IMessage>>> it = groupBy.entrySet().iterator();
- for (String groupByKey : sortKeys) {
+ Map<String, List<WindowValue>> temp = windowValues.stream().map((value) -> (WindowValue) value).collect(Collectors.groupingBy(WindowValue::getMsgKey));
+
+ Map<String, List<WindowValue>> groupByMsgKey = new HashMap<>(temp);
+
+ List<WindowValue> allWindowValues = new ArrayList<>();
+
+ //处理不同groupBy的message
+ for (String groupByKey : sortKeys) {
List<IMessage> msgs = groupBy.get(groupByKey);
String storeKey = createStoreKey(queueId, groupByKey, instance);
- WindowValue windowValue = (WindowValue) existWindowValues.get(storeKey);
- ;
- if (windowValue == null) {
- windowvaluecount++;
+
+ //msgKey 为唯一键
+ List<WindowValue> windowValueList = groupByMsgKey.get(storeKey);
+ WindowValue windowValue;
+ if (windowValueList == null || windowValueList.size() == 0) {
windowValue = createWindowValue(queueId, groupByKey, instance);
- // windowValue.setOrigOffset(msgs.get(0).getHeader().getOffset());
+ } else {
+ windowValue = windowValueList.get(0);
}
- allWindowValues.put(storeKey, windowValue);
- windowValue.incrementUpdateVersion();
- Integer origValue = getValue(windowValue, "total");
+ allWindowValues.add(windowValue);
+ windowValue.incrementUpdateVersion();
if (msgs != null) {
for (IMessage message : msgs) {
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java
index 9c1c0af6,5eb3b784..f356a286
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java
@@@ -60,11 -62,16 +60,10 @@@ public class JoinWindow extends Abstrac
protected String joinType;//join类型，值为INNER,LEFT
protected String expression;//条件表达式。在存在非等值比较时使用
- protected transient DBOperator joinOperator = new DBOperator();
- protected String rightDependentTableName;
- // @Override
- // protected void addPropertyToMessage(IMessage oriMessage, JSONObject oriJson){
- // oriJson.put("AbstractWindow", this);
- //
- // }
-
@Override
- protected int fireWindowInstance(WindowInstance instance, String shuffleId, Map<String, String> queueId2Offsets) {
+ protected int doFireWindowInstance(WindowInstance instance) {
+ //todo 只是清理吗？
clearFire(instance);
return 0;
}
@@@ -359,8 -422,18 +358,8 @@@
}
@Override
- protected String generateShuffleKey(IMessage message) {
+ public String generateShuffleKey(IMessage message) {
- String routeLabel =null;
- String lable = message.getHeader().getMsgRouteFromLable();
- if (lable != null) {
- if (lable.equals(rightDependentTableName)) {
- routeLabel = MessageHeader.JOIN_RIGHT;
- } else {
- routeLabel = MessageHeader.JOIN_LEFT;
- }
- } else {
- throw new RuntimeException("can not dipatch message, need route label " + toJson());
- }
+ String routeLabel = message.getHeader().getMsgRouteFromLable();
String messageKey = generateKey(message.getMessageBody(), routeLabel, leftJoinFieldNames, rightJoinFieldNames);
return messageKey;
}
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/AbstractSystemChannel.java
index 6c4659b3,d0e499f2..0a3845fd
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/AbstractSystemChannel.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/AbstractSystemChannel.java
@@@ -54,11 -55,12 +55,12 @@@ public abstract class AbstractSystemCha
protected static final String CHANNEL_PROPERTY_KEY_PREFIX = "CHANNEL_PROPERTY_KEY_PREFIX";
protected static final String CHANNEL_TYPE = "CHANNEL_TYPE";
- protected ISource consumer;
+ protected ISource<?> consumer;
protected AbstractSupportShuffleSink producer;
protected Map<String, String> channelConfig = new HashMap<>();
- protected volatile boolean hasCreateShuffleChannel = false;
+ protected boolean hasCreateShuffleChannel = false;
+ protected transient AtomicBoolean hasStart = new AtomicBoolean(false);
public void startChannel() {
if (consumer == null) {
return;
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleCache.java
index 5139f707,054a418d..7b4d8b4f
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleCache.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleCache.java
@@@ -22,11 -22,11 +22,11 @@@ import java.util.Collections
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.Future;
-
import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.rocketmq.streams.common.channel.sink.AbstractSink;
import org.apache.rocketmq.streams.common.context.IMessage;
-import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.apache.rocketmq.streams.common.context.MessageOffset;
import org.apache.rocketmq.streams.window.debug.DebugWriter;
import org.apache.rocketmq.streams.window.model.WindowCache;
import org.apache.rocketmq.streams.window.model.WindowInstance;
@@@ -36,9 -37,8 +36,9 @@@ import org.apache.rocketmq.streams.wind
/**
* save receiver messages into cachefilter when checkpoint/autoflush/flush， process cachefilter message
*/
- public class ShuffleCache extends WindowCache {
+ public class ShuffleCache extends AbstractSink {
protected AbstractShuffleWindow window;
+ private HashMap<String, Boolean> hasLoad = new HashMap<>();
public ShuffleCache(AbstractShuffleWindow window) {
this.window = window;
@@@ -64,64 -54,23 +64,65 @@@
for (Pair<String, String> queueIdAndInstanceKey : keys) {
String queueId = queueIdAndInstanceKey.getLeft();
String windowInstanceId = queueIdAndInstanceKey.getRight();
+
List<IMessage> messages = instance2Messages.get(queueIdAndInstanceKey);
+
WindowInstance windowInstance = windowInstanceMap.get(windowInstanceId);
+
DebugWriter.getDebugWriter(window.getConfigureName()).writeShuffleReceive(window, messages, windowInstance);
+
+ stateMustLoad(queueId);
+
window.shuffleCalculate(messages, windowInstance, queueId);
+
+ //保存处理进度
saveSplitProgress(queueId, messages);
+ window.saveMsgContext(queueId,windowInstance,messages);
}
- return true;
+ return true;
}
+ private void stateMustLoad(String queueId) {
+ Boolean load = this.hasLoad.get(queueId);
+ if (load != null && load) {
+ return;
+ }
+
+ //在计算之前需要异步加载状态完成
+ HashMap<String, Future<?>> loadResult = this.window.getShuffleChannel().getLoadResult();
+ Future<?> future = loadResult.get(queueId);
+
+ if (future == null) {
+ return;
+ }
+
+ try {
+ long before = System.currentTimeMillis();
+ future.get();
+ long after = System.currentTimeMillis();
+
+ System.out.println("message wait before state recover:[" + (after - before) + "] ms, queueId=" + queueId);
+
+ for (String loadQueueId : loadResult.keySet()) {
+ hasLoad.put(loadQueueId, true);
+ }
+ } catch (Throwable t) {
+ throw new RuntimeException("check remote with queueId:" + queueId + ",error", t);
+ }
+ }
+
/**
- * save consumer progress（offset）for groupby source queueId
+ * save consumer progress（offset）for groupby source shuffleId
+ * window configName: name_window_10001
+ * shuffleId: shuffle_NormalTestTopic_namespace_name_broker-a_001
+ * oriQueueId: NormalTestTopic2_broker-a_000
*
- * @param queueId
+ * @param shuffleId
* @param messages
*/
- protected void saveSplitProgress(String queueId, List<IMessage> messages) {
+ protected void saveSplitProgress(String shuffleId, List<IMessage> messages) {
+ IStorage delegator = this.window.getStorage();
+
Map<String, String> queueId2OrigOffset = new HashMap<>();
Boolean isLong = false;
for (IMessage message : messages) {
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
index f9bd3413,9280a2e4..d834af41
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
@@@ -18,7 -18,18 +18,8 @@@ package org.apache.rocketmq.streams.win
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang3.StringUtils;
+ import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.rocketmq.streams.common.batchsystem.BatchFinishMessage;
@@@ -42,7 -57,10 +46,8 @@@ import org.apache.rocketmq.streams.comm
import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
import org.apache.rocketmq.streams.common.utils.StringUtil;
import org.apache.rocketmq.streams.common.utils.TraceUtil;
-import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
import org.apache.rocketmq.streams.window.debug.DebugWriter;
+ import org.apache.rocketmq.streams.window.minibatch.MiniBatchMsgCache;
import org.apache.rocketmq.streams.window.model.WindowCache;
import org.apache.rocketmq.streams.window.model.WindowInstance;
import org.apache.rocketmq.streams.window.operator.AbstractShuffleWindow;
@@@ -77,11 -90,17 +85,13 @@@ public class ShuffleChannel extends Abs
protected ShuffleCache shuffleCache;
- protected Map<String, ISplit> queueMap = new ConcurrentHashMap<>();
- protected List<ISplit> queueList;//所有的分片
+ protected Map<String, ISplit<?, ?>> queueMap = new ConcurrentHashMap<>();
+ /**
+ * 所有的分片
+ */
+ protected List<ISplit<?, ?>> queueList;
- // protected NotifyChannel notfiyChannel;//负责做分片的通知管理
protected AbstractShuffleWindow window;
- /**
- * 当前管理的分片
- */
- private Set<String> currentQueueIds;
protected transient boolean isWindowTest = false;
@@@ -116,28 -126,16 +117,16 @@@
* init shuffle channel
*/
public void init() {
- this.consumer = createSource(window.getNameSpace(), window.getConfigureName());
-
- this.producer = createSink(window.getNameSpace(), window.getConfigureName());
- if (this.consumer == null || this.producer == null) {
- autoCreateShuffleChannel(window.getFireReceiver().getPipeline());
- }
- if (this.consumer == null) {
- return;
- }
- if (this.consumer instanceof AbstractSource) {
- ((AbstractSource) this.consumer).setJsonData(true);
- }
+ init(this.window);
if (producer != null && (queueList == null || queueList.size() == 0)) {
queueList = producer.getSplitList();
- Map<String, ISplit> tmp = new ConcurrentHashMap<>();
- for (ISplit queue : queueList) {
+ Map<String, ISplit<?, ?>> tmp = new ConcurrentHashMap<>();
+ for (ISplit<?, ?> queue : queueList) {
tmp.put(queue.getQueueId(), queue);
}
-
this.queueMap = tmp;
}
- isWindowTest = ComponentCreator.getPropertyBooleanValue("window.fire.isTest");
+// isWindowTest = ComponentCreator.getPropertyBooleanValue("window.fire.isTest");
}
/**
@@@ -467,8 -505,11 +455,8 @@@
return splitNum > 0 ? splitNum : 32;
}
- public Set<String> getCurrentQueueIds() {
- return currentQueueIds;
- }
- public List<ISplit> getQueueList() {
+ public List<ISplit<?, ?>> getQueueList() {
return queueList;
}
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/trigger/WindowTrigger.java
index b012b175,69b6494a..e3bb3f83
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/trigger/WindowTrigger.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/trigger/WindowTrigger.java
@@@ -287,7 -287,7 +287,6 @@@ public class WindowTrigger extends Abst
}
});
for (WindowInstance windowInstance : windowInstanceList) {
- System.out.println("fire by finish flag");
- // System.out.println("fire by finish flag");
fireWindowInstance(windowInstance);
}
}
[rocketmq-streams] 16/16: merge from replaceDB
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 42abb2099f8fabad84939454fa765351dbbd1acc
Merge: aadf7eaa d2464881
Author: 维章 <un...@gmail.com>
AuthorDate: Tue May 31 11:03:46 2022 +0800
merge from replaceDB
.../org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java | 1 -
.../apache/rocketmq/streams/window/storage/rocketmq/DefaultStorage.java | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
[rocketmq-streams] 12/16: fix(example) join example
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 5a236975a497a970c43f5b576ece46b5502812cb
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 30 15:10:28 2022 +0800
fix(example) join example
---
.../apache/rocketmq/streams/examples/join/RocketmqJoinExample.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
index f4270d8a..30dec798 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
@@ -22,9 +22,9 @@ import org.apache.rocketmq.streams.client.transform.DataStream;
public class RocketmqJoinExample {
public static void main(String[] args) {
DataStream left = StreamBuilder.dataStream("tmp", "tmp")
- .fromRocketmq("TopicTest", "groupA", true, "localhost:9876");
- DataStream right = StreamBuilder.dataStream("tmp", "tmp")
- .fromRocketmq("TopicTest", "groupB", true, "localhost:9876");
+ .fromRocketmq("TopicTestJoin", "groupA", true, "localhost:9876");
+ DataStream right = StreamBuilder.dataStream("tmp", "tmp22")
+ .fromRocketmq("TopicTestJoin", "groupB", true, "localhost:9876");
left.join(right)
.on("(ProjectName,=,ProjectName)")
[rocketmq-streams] 03/16: merge 1.0
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 9d3ae58bf1f301a7f31d04256ddbe04ec94899b5
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 23 11:58:39 2022 +0800
merge 1.0
---
pom.xml | 1 +
rocketmq-streams-connectors/pom.xml | 47 ++++
.../streams/connectors/IBoundedSource.java | 32 +++
.../streams/connectors/IBoundedSourceReader.java | 26 ++
.../streams/connectors/IScheduleCallback.java | 24 ++
.../connectors/balance/AbstractBalance.java | 207 ++++++++++++++
.../streams/connectors/balance/IBalanceTask.java | 24 ++
.../streams/connectors/balance/ISourceBalance.java | 60 ++++
.../streams/connectors/balance/SplitChanged.java | 55 ++++
.../connectors/balance/impl/LeaseBalanceImpl.java | 144 ++++++++++
.../streams/connectors/model/PullMessage.java | 50 ++++
.../streams/connectors/model/ReaderStatus.java | 120 ++++++++
.../streams/connectors/reader/DBScanReader.java | 269 ++++++++++++++++++
.../streams/connectors/reader/ISplitReader.java | 96 +++++++
.../connectors/reader/SplitCloseFuture.java | 83 ++++++
.../connectors/source/AbstractPullSource.java | 312 +++++++++++++++++++++
.../source/CycleDynamicMultipleDBScanSource.java | 213 ++++++++++++++
.../source/DynamicMultipleDBScanSource.java | 190 +++++++++++++
.../streams/connectors/source/IPullSource.java | 60 ++++
.../connectors/source/MutilBatchTaskSource.java | 158 +++++++++++
.../streams/connectors/source/SourceInstance.java | 37 +++
.../source/filter/AbstractPatternFilter.java | 38 +++
.../source/filter/BoundedPatternFilter.java | 53 ++++
.../source/filter/CyclePatternFilter.java | 173 ++++++++++++
.../connectors/source/filter/CyclePeriod.java | 222 +++++++++++++++
.../connectors/source/filter/CycleSchedule.java | 236 ++++++++++++++++
.../source/filter/CycleScheduleFilter.java | 37 +++
.../source/filter/DataFormatPatternFilter.java | 106 +++++++
.../connectors/source/filter/PatternFilter.java | 41 +++
.../window/offset/WindowMaxValueProcessor.java | 2 +-
30 files changed, 3115 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 9da26dcd..7caa0cbc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -52,6 +52,7 @@
<module>rocketmq-streams-channel-http</module>
<module>rocketmq-streams-state</module>
<module>rocketmq-streams-examples</module>
+ <module>rocketmq-streams-connectors</module>
<module>rocketmq-streams-channel-syslog</module>
<module>rocketmq-streams-channel-es</module>
<module>rocketmq-streams-channel-kafka</module>
diff --git a/rocketmq-streams-connectors/pom.xml b/rocketmq-streams-connectors/pom.xml
new file mode 100644
index 00000000..d544e3d5
--- /dev/null
+++ b/rocketmq-streams-connectors/pom.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>1.0.2-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-connectors</artifactId>
+ <packaging>jar</packaging>
+ <name>ROCKETMQ STREAMS :: connectors</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-lease</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-schedule</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-commons</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IBoundedSource.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IBoundedSource.java
new file mode 100644
index 00000000..1a383433
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IBoundedSource.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors;
+
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+
+/**
+ * @description
+ */
+public interface IBoundedSource{
+
+ /**
+ * reader完成时调用
+ * @param iSplit
+ */
+ void boundedFinishedCallBack(ISplit iSplit);
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IBoundedSourceReader.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IBoundedSourceReader.java
new file mode 100644
index 00000000..03995c31
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IBoundedSourceReader.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors;
+
+import org.apache.rocketmq.streams.common.interfaces.ILifeCycle;
+
+/**
+ * @description
+ */
+public interface IBoundedSourceReader extends ILifeCycle {
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IScheduleCallback.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IScheduleCallback.java
new file mode 100644
index 00000000..88fb2cfd
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/IScheduleCallback.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors;
+
+import java.util.Date;
+
+public interface IScheduleCallback {
+
+ boolean canFire(Date date);
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/AbstractBalance.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/AbstractBalance.java
new file mode 100644
index 00000000..5c2b266f
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/AbstractBalance.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.balance;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.connectors.source.SourceInstance;
+
+public abstract class AbstractBalance implements ISourceBalance {
+
+ protected int balanceCount = 0;
+
+ @Override
+ public SplitChanged doBalance(List<ISplit> allSplits, List<ISplit> ownerSplits) {
+ balanceCount++;
+ heartBeat();
+ List<SourceInstance> sourceInstances = fetchSourceInstances();
+ List<ISplit> workingSplits = fetchWorkingSplits(allSplits);
+ SplitChanged splitChanged = getAdditionSplits(allSplits, sourceInstances, workingSplits, ownerSplits);
+ if (splitChanged != null) {
+ return splitChanged;
+ }
+ splitChanged = getRemoveSplits(allSplits, sourceInstances, workingSplits, ownerSplits);
+ return splitChanged;
+ }
+
+ protected void heartBeat() {
+ holdLockSourceInstance();
+ }
+
+ /**
+ * get all dispatch splits
+ *
+ * @return
+ */
+ protected abstract List<ISplit> fetchWorkingSplits(List<ISplit> allSplitS);
+
+ /**
+ * get all instacne for the source
+ *
+ * @return
+ */
+ protected abstract List<SourceInstance> fetchSourceInstances();
+
+ /**
+ * lock the source ,the lock is globel,only one source instance can get it in same time
+ *
+ * @return
+ */
+ protected abstract boolean holdLockSourceInstance();
+
+ /**
+ * unlock
+ */
+ protected abstract void unlockSourceInstance();
+
+ /**
+ * juge need add split，根据调度策略选择
+ * 每次最大增加的分片数，根据调度次数决定
+ *
+ * @param allSplits
+ * @param sourceInstances
+ * @param workingSplits
+ * @return
+ */
+ protected SplitChanged getAdditionSplits(List<ISplit> allSplits, List<SourceInstance> sourceInstances,
+ List<ISplit> workingSplits, List<ISplit> ownerSplits) {
+ SplitChanged splitChanged = getChangedSplitCount(allSplits, sourceInstances, workingSplits.size(), ownerSplits.size());
+ if (splitChanged == null) {
+ return null;
+ }
+ if (splitChanged.isNewSplit == false) {
+ return null;
+ }
+ if (splitChanged.splitCount <= 0) {
+ return null;
+ }
+ List<ISplit> noWorkingSplits = getNoWorkingSplits(allSplits, workingSplits);
+ List<ISplit> newSplits = new ArrayList<>();
+ for (int i = 0; i < noWorkingSplits.size(); i++) {
+ boolean success = holdLockSplit(noWorkingSplits.get(i));
+ if (success) {
+ newSplits.add(noWorkingSplits.get(i));
+ if (newSplits.size() >= splitChanged.splitCount) {
+ break;
+ }
+ }
+ }
+ splitChanged.setChangedSplits(newSplits);
+ return splitChanged;
+
+ }
+
+ protected List<ISplit> getNoWorkingSplits(List<ISplit> allSplits, List<ISplit> workingSplits) {
+ Set<String> workingSplitIds = new HashSet<>();
+ for (ISplit split : workingSplits) {
+ workingSplitIds.add(split.getQueueId());
+ }
+ List<ISplit> splits = new ArrayList<>();
+ for (ISplit split : allSplits) {
+ if (!workingSplitIds.contains(split.getQueueId())) {
+ splits.add(split);
+ }
+ }
+ return splits;
+ }
+
+ /**
+ * 获取需要删除的分片
+ *
+ * @param allSplits
+ * @param sourceInstances
+ * @param workingSplits
+ * @return
+ */
+ protected SplitChanged getRemoveSplits(List<ISplit> allSplits, List<SourceInstance> sourceInstances,
+ List<ISplit> workingSplits, List<ISplit> ownerSplits) {
+ SplitChanged splitChanged = getChangedSplitCount(allSplits, sourceInstances, workingSplits.size(), ownerSplits.size());
+ if (splitChanged == null) {
+ return null;
+ }
+ if (splitChanged.isNewSplit == true) {
+ return null;
+ }
+
+ if (splitChanged.splitCount <= 0) {
+ return null;
+ }
+ //List<ISplit> ownerSplits=source.ownerSplits();
+ List<ISplit> removeSplits = new ArrayList();
+ for (int i = 0; i < splitChanged.splitCount; i++) {
+ removeSplits.add(ownerSplits.get(i));
+ }
+ splitChanged.setChangedSplits(removeSplits);
+ return splitChanged;
+ }
+
+ /**
+ * 获取需要变动的分片个数，新增或删除
+ * 分配策略，只有有未分配的分片时才会分配新分片，为了减少分片切换，前面几次尽可能少分，后面越来越多
+ *
+ * @return 需要本实例有变化的分配，新增或删除
+ */
+ protected SplitChanged getChangedSplitCount(List<ISplit> allSplits, List<SourceInstance> sourceInstances,
+ int splitCountInWorking, int ownerSplitCount) {
+ //int ownerSplitCount=source.ownerSplits().size();
+ int instanceCount = sourceInstances.size();
+ if (instanceCount == 0) {
+ instanceCount = 1;
+ }
+ int allSplitCount = allSplits.size();
+ int minSplitCount = allSplitCount / instanceCount;
+ int maxSplitCount = minSplitCount + (allSplitCount % instanceCount == 0 ? 0 : 1);
+ //已经是最大分片数了
+ if (ownerSplitCount == maxSplitCount) {
+ return null;
+ }
+ if (ownerSplitCount > maxSplitCount) {
+ int changeSplitCount = ownerSplitCount - maxSplitCount;
+ return new SplitChanged(changeSplitCount, false);
+ }
+ //分片已经全部在处理，当前分片也符合最小分片分配策略，不需要重新分配
+ if (splitCountInWorking == allSplitCount && ownerSplitCount >= minSplitCount) {
+ return null;
+ }
+ //如果还有未分配的分片，且当前实例还有分片的可行性，则分配分片
+ if (splitCountInWorking < allSplitCount && ownerSplitCount < maxSplitCount) {
+ int changeSplitCount = Math.min(maxSplitCount - ownerSplitCount, getMaxSplitCountInOneBalance());
+
+ return new SplitChanged(changeSplitCount, true);
+ }
+ return null;
+ }
+
+ @Override
+ public int getBalanceCount() {
+ return balanceCount;
+ }
+
+ /**
+ * 每次负载均衡最大的分片个数,目的是前几次，少分配分配，可能有实例在启动中，以免频繁切换分片，到后面实例都启动了，斤可能多分配分片
+ *
+ * @return
+ */
+ private int getMaxSplitCountInOneBalance() {
+ int balanceCount = getBalanceCount();
+ return (int) Math.pow(2, balanceCount - 1);
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/IBalanceTask.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/IBalanceTask.java
new file mode 100644
index 00000000..5275fe48
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/IBalanceTask.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.balance;
+
+/**
+ * @description
+ */
+public interface IBalanceTask extends Runnable {
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/ISourceBalance.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/ISourceBalance.java
new file mode 100644
index 00000000..b012b323
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/ISourceBalance.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.balance;
+
+import java.util.List;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+
+public interface ISourceBalance {
+
+ /**
+ * 做负载均衡
+
+ * @return
+ */
+ SplitChanged doBalance(List<ISplit> allSplits, List<ISplit> ownerSplits);
+
+ /**
+ * 从启动开始，做了多少次均衡
+ * @return
+ */
+ int getBalanceCount();
+
+
+
+ boolean getRemoveSplitLock();
+
+ void unLockRemoveSplitLock();
+
+ /**
+ * lock the split and hold it util the instance is shutdown or remove split
+ * @param split
+ * @return
+ */
+ boolean holdLockSplit(ISplit split);
+
+ /**
+ * unlock split lock
+ * @param split
+ */
+ void unlockSplit(ISplit split);
+
+
+ void setSourceIdentification(String sourceIdentification);
+
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/SplitChanged.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/SplitChanged.java
new file mode 100644
index 00000000..c01c1519
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/SplitChanged.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.balance;
+
+import java.util.List;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+
+public class SplitChanged {
+
+ protected int splitCount;//变动多分片个数
+ protected boolean isNewSplit;//是否新增，false是删除
+ protected List<ISplit> changedSplits;
+ public SplitChanged(int splitCount,boolean isNewSplit){
+ this.splitCount=splitCount;
+ this.isNewSplit=isNewSplit;
+ }
+
+ public int getSplitCount() {
+ return splitCount;
+ }
+
+ public void setSplitCount(int splitCount) {
+ this.splitCount = splitCount;
+ }
+
+ public boolean isNewSplit() {
+ return isNewSplit;
+ }
+
+ public void setNewSplit(boolean newSplit) {
+ isNewSplit = newSplit;
+ }
+
+ public List<ISplit> getChangedSplits() {
+ return changedSplits;
+ }
+
+ public void setChangedSplits(List<ISplit> changedSplits) {
+ this.changedSplits = changedSplits;
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/impl/LeaseBalanceImpl.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/impl/LeaseBalanceImpl.java
new file mode 100644
index 00000000..dc504e5d
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/impl/LeaseBalanceImpl.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.balance.impl;
+
+import com.google.auto.service.AutoService;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.RuntimeUtil;
+import org.apache.rocketmq.streams.connectors.balance.AbstractBalance;
+import org.apache.rocketmq.streams.connectors.balance.ISourceBalance;
+import org.apache.rocketmq.streams.connectors.source.SourceInstance;
+import org.apache.rocketmq.streams.lease.LeaseComponent;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+
+@AutoService(ISourceBalance.class)
+@ServiceName(LeaseBalanceImpl.DB_BALANCE_NAME)
+public class LeaseBalanceImpl extends AbstractBalance {
+
+ private static final Log logger = LogFactory.getLog(LeaseBalanceImpl.class);
+
+ public static final String DB_BALANCE_NAME = "db_balance";
+ private static final String REMOVE_SPLIT_LOCK_NAME = "lock_remove_split";
+ private static final String SOURCE_LOCK_PREFIX = "SOURCE_";
+ private static final String SPLIT_LOCK_PREFIX = "SPLIT_";
+ protected transient LeaseComponent leaseComponent = LeaseComponent.getInstance();
+ protected transient String sourceIdentification;
+
+ protected int lockTimeSecond = 5;
+
+ public LeaseBalanceImpl(String sourceIdentification) {
+
+ this.sourceIdentification = sourceIdentification;
+ }
+
+ public LeaseBalanceImpl() {
+
+ }
+
+ @Override
+ protected List<ISplit> fetchWorkingSplits(List<ISplit> allSplits) {
+ List<LeaseInfo> leaseInfos = leaseComponent.getService().queryLockedInstanceByNamePrefix(SPLIT_LOCK_PREFIX + this.sourceIdentification, null);
+ logger.info(String.format("lease SPLIT_LOCK_PREFIX is %s, sourceIdentification is %s. ", SPLIT_LOCK_PREFIX, sourceIdentification));
+ if (leaseInfos == null) {
+ return new ArrayList<>();
+ }
+
+ Map<String, ISplit> allSplitMap = new HashMap<>();
+ for (ISplit split : allSplits) {
+ allSplitMap.put(split.getQueueId(), split);
+ }
+ List<ISplit> splits = new ArrayList<>();
+ for (LeaseInfo leaseInfo : leaseInfos) {
+ String leaseName = leaseInfo.getLeaseName();
+ String splitId = MapKeyUtil.getLast(leaseName);
+ splits.add(allSplitMap.get(splitId));
+ }
+ logger.info(String.format("working split is %s", Arrays.toString(splits.toArray())));
+ return splits;
+ }
+
+ @Override
+ protected List<SourceInstance> fetchSourceInstances() {
+ List<LeaseInfo> leaseInfos = leaseComponent.getService().queryLockedInstanceByNamePrefix(SOURCE_LOCK_PREFIX + sourceIdentification, null);
+ if (leaseInfos == null) {
+ return new ArrayList<>();
+ }
+ List<SourceInstance> sourceInstances = new ArrayList<>();
+ for (LeaseInfo leaseInfo : leaseInfos) {
+ String leaseName = leaseInfo.getLeaseName();
+ sourceInstances.add(new SourceInstance(leaseName));
+ }
+ return sourceInstances;
+ }
+
+ @Override
+ protected boolean holdLockSourceInstance() {
+ return holdLock(SOURCE_LOCK_PREFIX + sourceIdentification, RuntimeUtil.getDipperInstanceId());
+ }
+
+ @Override
+ protected void unlockSourceInstance() {
+ leaseComponent.getService().unlock(SOURCE_LOCK_PREFIX + sourceIdentification, RuntimeUtil.getDipperInstanceId());
+ }
+
+ @Override
+ public boolean holdLockSplit(ISplit split) {
+ return holdLock(SPLIT_LOCK_PREFIX + this.sourceIdentification, split.getQueueId());
+ }
+
+ @Override
+ public void unlockSplit(ISplit split) {
+ leaseComponent.getService().unlock(SPLIT_LOCK_PREFIX + this.sourceIdentification, split.getQueueId());
+
+ }
+
+ @Override
+ public boolean getRemoveSplitLock() {
+ return holdLock(this.sourceIdentification, REMOVE_SPLIT_LOCK_NAME);
+ }
+
+ @Override
+ public void unLockRemoveSplitLock() {
+ leaseComponent.getService().unlock(this.sourceIdentification, REMOVE_SPLIT_LOCK_NAME);
+ }
+
+ public String getSourceIdentification() {
+ return sourceIdentification;
+ }
+
+ @Override
+ public void setSourceIdentification(String sourceIdentification) {
+ this.sourceIdentification = sourceIdentification;
+ }
+
+ protected boolean holdLock(String name, String lockName) {
+ ILeaseService leaseService = leaseComponent.getService();
+ boolean success = leaseService.holdLock(name, lockName, lockTimeSecond);
+ return success;
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/PullMessage.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/PullMessage.java
new file mode 100644
index 00000000..9bf34803
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/PullMessage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.model;
+
+import org.apache.rocketmq.streams.common.context.MessageOffset;
+
+public class PullMessage<T> {
+ protected T message;
+ protected MessageOffset messageOffset;
+
+ public T getMessage() {
+ return message;
+ }
+
+ public void setMessage(T message) {
+ this.message = message;
+ }
+
+ public MessageOffset getMessageOffset() {
+ return messageOffset;
+ }
+
+ public void setMessageOffset(MessageOffset messageOffset) {
+ this.messageOffset = messageOffset;
+ }
+ /**
+ * 获取offset字符串，通过.把主offset和子offset串接在一起
+ * @return
+ */
+ public String getOffsetStr(){
+ return this.messageOffset.getOffsetStr();
+ }
+ public String getMainOffset() {
+ return messageOffset.getMainOffset();
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/ReaderStatus.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/ReaderStatus.java
new file mode 100644
index 00000000..a4889b5f
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/ReaderStatus.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.model;
+
+import java.util.Date;
+import java.util.List;
+import org.apache.rocketmq.streams.common.model.Entity;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+
+/**
+ * @description
+ */
+public class ReaderStatus extends Entity {
+
+ /**
+ * 查询单个readerStatus
+ */
+ static final String queryReaderStatusByUK = "select * from reader_status where source_name = '%s' and reader_name = '%s' and is_finished = 1";
+
+ static final String queryReaderStatusList = "select * from reader_status where source_name = '%s' and is_finished = 1";
+
+ static final String clearReaderStatus = "update reader_status set gmt_modified = now(), is_finished = -1 where source_name = '%s' and reader_name = '%s'";
+
+ String sourceName;
+
+ String readerName;
+
+ int isFinished;
+
+ int totalReader;
+
+ public String getReaderName() {
+ return readerName;
+ }
+
+ public void setReaderName(String readerName) {
+ this.readerName = readerName;
+ }
+
+ public int getIsFinished() {
+ return isFinished;
+ }
+
+ public void setIsFinished(int isFinished) {
+ this.isFinished = isFinished;
+ }
+
+ public int getTotalReader() {
+ return totalReader;
+ }
+
+ public void setTotalReader(int totalReader) {
+ this.totalReader = totalReader;
+ }
+
+ public String getSourceName() {
+ return sourceName;
+ }
+
+ public void setSourceName(String sourceName) {
+ this.sourceName = sourceName;
+ }
+
+ @Override
+ public String toString() {
+ return "ReaderStatus{" +
+ "id=" + id +
+ ", gmtCreate=" + gmtCreate +
+ ", gmtModified=" + gmtModified +
+ ", sourceName='" + sourceName + '\'' +
+ ", readerName='" + readerName + '\'' +
+ ", isFinished=" + isFinished +
+ ", totalReader=" + totalReader +
+ '}';
+ }
+
+ public static ReaderStatus queryReaderStatusByUK(String sourceName, String readerName) {
+ String sql = String.format(queryReaderStatusByUK, sourceName, readerName);
+ ReaderStatus readerStatus = ORMUtil.queryForObject(sql, null, ReaderStatus.class);
+ return readerStatus;
+ }
+
+ public static List<ReaderStatus> queryReaderStatusListBySourceName(String sourceName) {
+ String sql = String.format(queryReaderStatusList, sourceName);
+ List<ReaderStatus> readerStatusList = ORMUtil.queryForList(sql, null, ReaderStatus.class);
+ return readerStatusList;
+ }
+
+ public static void clearReaderStatus(String sourceName, String readerName) {
+ String sql = String.format(clearReaderStatus, sourceName, readerName);
+ ORMUtil.executeSQL(sql, null);
+ }
+
+ public static ReaderStatus create(String sourceName, String readerName, int isFinished, int totalReader) {
+
+ ReaderStatus readerStatus = new ReaderStatus();
+ readerStatus.setSourceName(sourceName);
+ readerStatus.setReaderName(readerName);
+ readerStatus.setIsFinished(isFinished);
+ readerStatus.setTotalReader(totalReader);
+ readerStatus.setGmtCreate(new Date());
+ readerStatus.setGmtModified(new Date());
+ return readerStatus;
+
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/DBScanReader.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/DBScanReader.java
new file mode 100644
index 00000000..268e891e
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/DBScanReader.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.reader;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.channel.source.ISource;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.context.MessageOffset;
+import org.apache.rocketmq.streams.common.utils.ThreadUtil;
+import org.apache.rocketmq.streams.connectors.IBoundedSource;
+import org.apache.rocketmq.streams.connectors.IBoundedSourceReader;
+import org.apache.rocketmq.streams.connectors.model.PullMessage;
+import org.apache.rocketmq.streams.connectors.model.ReaderStatus;
+import org.apache.rocketmq.streams.connectors.source.CycleDynamicMultipleDBScanSource;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+
+/**
+ * @description
+ */
+public class DBScanReader implements ISplitReader, IBoundedSourceReader, Serializable {
+
+ private static final long serialVersionUID = 8172403250050893288L;
+ private static final Log logger = LogFactory.getLog(DBScanReader.class);
+ static final String sqlTemplate = "select * from %s where id >= %d and id < %d";
+
+ //是否完成了source的call back调用
+ transient volatile boolean isFinishedCall = false;
+ ISource iSource;
+ String url;
+ String userName;
+ String password;
+ String tableName;
+ int batchSize;
+ long offset;
+ long offsetStart;
+ long offsetEnd;
+ long maxOffset;
+ long minOffset;
+ ISplit iSplit;
+ transient List<PullMessage> pullMessages;
+ volatile boolean interrupt = false;
+ volatile boolean isClosed = false;
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public ISplit getISplit() {
+ return iSplit;
+ }
+
+ public void setISplit(ISplit iSplit) {
+ this.iSplit = iSplit;
+ }
+
+ public DBScanReader() {
+
+ }
+
+ transient ThreadLocal<JDBCDriver> threadLocal = new ThreadLocal<JDBCDriver>() {
+
+ @Override
+ public JDBCDriver initialValue() {
+ logger.info(String.format("%s initial jdbcDriver. ", Thread.currentThread().getName()));
+ return DriverBuilder.createDriver(AbstractComponent.DEFAULT_JDBC_DRIVER, url, userName, password);
+ }
+
+ };
+
+ @Override
+ public void open(ISplit split) {
+ this.iSplit = split;
+ JDBCDriver jdbcDriver = threadLocal.get();
+ Map<String, Object> range = jdbcDriver.queryOneRow("select min(id) as min_id, max(id) as max_id from " + tableName);
+ minOffset = Long.parseLong(String.valueOf(range.get("min_id")));
+ maxOffset = Long.parseLong(String.valueOf(range.get("max_id")));
+ offsetStart = minOffset;
+ offset = minOffset;
+ logger.info(String.format("table %s min id [ %d ], max id [ %d ]", tableName, minOffset, maxOffset));
+ pullMessages = new ArrayList<>();
+ }
+
+ @Override
+ public boolean next() {
+ if (interrupt) {
+ return false;
+ }
+ if (isFinished()) {
+ finish();
+ ThreadUtil.sleep(10 * 1000);
+ return false;
+ }
+ JDBCDriver jdbcDriver = threadLocal.get();
+ offsetEnd = offsetStart + batchSize;
+ String batchQuery = String.format(sqlTemplate, tableName, offsetStart, offsetEnd);
+ logger.debug(String.format("execute sql : %s", batchQuery));
+ List<Map<String, Object>> resultData = jdbcDriver.queryForList(batchQuery);
+ offsetStart = offsetEnd;
+ pullMessages.clear();
+ for (Map<String, Object> r : resultData) {
+ PullMessage msg = new PullMessage();
+ JSONObject data = JSONObject.parseObject(JSON.toJSONString(r));
+ msg.setMessage(data);
+ offset = offset > Long.parseLong(data.getString("id")) ? offset : Long.parseLong(data.getString("id"));
+ msg.setMessageOffset(new MessageOffset(String.valueOf(offset), true));
+ pullMessages.add(msg);
+ }
+ return offsetStart - batchSize <= maxOffset;
+ }
+
+ @Override
+ public List<PullMessage> getMessage() {
+// logger.info(String.format("output messages %d", pullMessages.size()));
+ return pullMessages;
+ }
+
+ @Override
+ public SplitCloseFuture close() {
+// interrupt = true;
+ isClosed = true;
+ threadLocal.remove();
+ pullMessages = null;
+ return new SplitCloseFuture(this, iSplit);
+ }
+
+ @Override
+ public void seek(String cursor) {
+ if (cursor == null || cursor.trim().equals("")) {
+ cursor = "0";
+ }
+ offset = Long.parseLong(cursor);
+ if (offset < minOffset) {
+ offset = minOffset;
+ }
+ offsetStart = offset;
+ logger.info(String.format("split %s seek %d.", iSplit.getQueueId(), offset));
+ }
+
+ @Override
+ public String getProgress() {
+ return String.valueOf(offset);
+ }
+
+ @Override
+ public long getDelay() {
+ return maxOffset - offset;
+ }
+
+ @Override
+ public long getFetchedDelay() {
+ return 0;
+ }
+
+ @Override
+ public boolean isClose() {
+ return isClosed;
+ }
+
+ @Override
+ public ISplit getSplit() {
+ return iSplit;
+ }
+
+ @Override
+ public boolean isInterrupt() {
+ return interrupt;
+ }
+
+ @Override
+ public boolean interrupt() {
+ interrupt = true;
+ return true;
+ }
+
+ @Override
+ public boolean isFinished() {
+ return offsetStart > maxOffset;
+ }
+
+ @Override
+ public void finish() {
+ if (isFinishedCall) {
+ return;
+ }
+ pullMessages = null;
+ updateReaderStatus();
+ IBoundedSource tmp = (IBoundedSource) iSource;
+ tmp.boundedFinishedCallBack(this.iSplit);
+ isFinishedCall = true;
+ }
+
+ public ISource getISource() {
+ return iSource;
+ }
+
+ public void setISource(ISource iSource) {
+ this.iSource = iSource;
+ }
+
+ private final void updateReaderStatus() {
+ String sourceName = CycleDynamicMultipleDBScanSource.createKey(this.getISource());
+ int finish = Integer.valueOf(1);
+ int total = ((CycleDynamicMultipleDBScanSource) iSource).getTotalReader();
+ ReaderStatus readerStatus = ReaderStatus.create(sourceName, iSplit.getQueueId(), finish, total);
+ logger.info(String.format("create reader status %s.", readerStatus));
+ ORMUtil.batchReplaceInto(readerStatus);
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/ISplitReader.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/ISplitReader.java
new file mode 100644
index 00000000..6b377cff
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/ISplitReader.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.reader;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.connectors.model.PullMessage;
+
+public interface ISplitReader extends Serializable {
+
+ /**
+ * Open.
+ *
+ * @param split the split
+ * @throws IOException the io exception
+ */
+ void open(ISplit split);
+
+ /**
+ * Next boolean.
+ *
+ * @return the boolean
+ * @throws IOException the io exception
+ * @throws InterruptedException the interrupted exception
+ */
+ boolean next();
+
+ /**
+ * Gets message.
+ *
+ * @return the message
+ */
+ List<PullMessage> getMessage();
+
+ /**
+ * Close.
+ *
+ * @throws IOException the io exception
+ */
+ SplitCloseFuture close();
+
+ /**
+ * Seek.
+ *
+ * @param cursor the cursor
+ * @throws IOException the io exception
+ */
+ void seek(String cursor);
+
+ /**
+ * Gets progress.
+ *
+ * @return the progress
+ * @throws IOException the io exception
+ */
+ String getProgress();
+
+ /**
+ * Get message delay (millseconds)
+ *
+ * @return delay
+ */
+ long getDelay();
+
+ /**
+ * Get message delay (millseconds) from being fetched
+ *
+ * @return delay
+ */
+ long getFetchedDelay();
+
+ boolean isClose();
+
+ ISplit getSplit();
+
+ boolean isInterrupt();
+
+ boolean interrupt();
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/SplitCloseFuture.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/SplitCloseFuture.java
new file mode 100644
index 00000000..b28748b8
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/SplitCloseFuture.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.reader;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+
+public class SplitCloseFuture implements Future<Boolean> {
+
+ protected ISplitReader reader;
+ protected ISplit split;
+
+ public SplitCloseFuture(ISplitReader reader, ISplit split) {
+ this.reader = reader;
+ this.split = split;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ return reader.isClose();
+ }
+
+ @Override
+ public Boolean get() throws InterruptedException, ExecutionException {
+ synchronized (reader) {
+ reader.wait();
+ }
+ return reader.isClose();
+ }
+
+ @Override
+ public Boolean get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ synchronized (reader) {
+ long time = timeout;
+ if (unit == TimeUnit.SECONDS) {
+ time = time * 1000;
+ } else if (unit == TimeUnit.MINUTES) {
+ time = time * 1000 * 60;
+ } else if (unit == TimeUnit.HOURS) {
+ time = time * 1000 * 60 * 60;
+ } else {
+ throw new RuntimeException("can not support this timeout, expect less hour " + timeout + " the unit is " + unit);
+ }
+ reader.wait(time);
+ }
+ return reader.isClose();
+ }
+
+ public ISplitReader getReader() {
+ return reader;
+ }
+
+ public ISplit getSplit() {
+ return split;
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/AbstractPullSource.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/AbstractPullSource.java
new file mode 100644
index 00000000..9aedc95a
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/AbstractPullSource.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.channel.source.AbstractSource;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.checkpoint.CheckPoint;
+import org.apache.rocketmq.streams.common.checkpoint.CheckPointManager;
+import org.apache.rocketmq.streams.common.context.Message;
+import org.apache.rocketmq.streams.common.threadpool.ThreadPoolFactory;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.connectors.balance.ISourceBalance;
+import org.apache.rocketmq.streams.connectors.balance.SplitChanged;
+import org.apache.rocketmq.streams.connectors.balance.impl.LeaseBalanceImpl;
+import org.apache.rocketmq.streams.connectors.model.PullMessage;
+import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
+import org.apache.rocketmq.streams.connectors.reader.SplitCloseFuture;
+import org.apache.rocketmq.streams.serviceloader.ServiceLoaderComponent;
+
+public abstract class AbstractPullSource extends AbstractSource implements IPullSource<AbstractSource> {
+
+ private static final Log logger = LogFactory.getLog(AbstractPullSource.class);
+
+ protected transient ISourceBalance balance;// balance interface
+ protected transient ScheduledExecutorService balanceExecutor;//schdeule balance
+ protected transient Map<String, ISplitReader> splitReaders = new HashMap<>();//owner split readers
+ protected transient Map<String, ISplit> ownerSplits = new HashMap<>();//working splits by the source instance
+
+ //可以有多种实现，通过名字选择不同的实现
+ protected String balanceName = LeaseBalanceImpl.DB_BALANCE_NAME;
+ //balance schedule time
+ protected int balanceTimeSecond = 10;
+ protected long pullIntervalMs;
+ protected transient CheckPointManager checkPointManager = new CheckPointManager();
+ protected transient boolean shutDown=false;
+ @Override
+ protected boolean startSource() {
+ ServiceLoaderComponent serviceLoaderComponent = ServiceLoaderComponent.getInstance(ISourceBalance.class);
+ balance = (ISourceBalance) serviceLoaderComponent.getService().loadService(balanceName);
+ balance.setSourceIdentification(MapKeyUtil.createKey(getNameSpace(), getConfigureName()));
+ balanceExecutor = new ScheduledThreadPoolExecutor(1, new BasicThreadFactory.Builder().namingPattern("balance-task-%d").daemon(true).build());
+ List<ISplit> allSplits = fetchAllSplits();
+ SplitChanged splitChanged = balance.doBalance(allSplits, new ArrayList(ownerSplits.values()));
+ doSplitChanged(splitChanged);
+ balanceExecutor.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ logger.info("balance running..... current splits is " + ownerSplits);
+ List<ISplit> allSplits = fetchAllSplits();
+ SplitChanged splitChanged = balance.doBalance(allSplits, new ArrayList(ownerSplits.values()));
+ doSplitChanged(splitChanged);
+ }
+ }, balanceTimeSecond, balanceTimeSecond, TimeUnit.SECONDS);
+
+ startWorks();
+ return true;
+ }
+
+ private void startWorks() {
+ ExecutorService workThreads= ThreadPoolFactory.createThreadPool(maxThread);
+ long start=System.currentTimeMillis();
+ while (!shutDown) {
+ Iterator<Map.Entry<String, ISplitReader>> it = splitReaders.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, ISplitReader> entry=it.next();
+ String splitId=entry.getKey();
+ ISplit split=ownerSplits.get(splitId);
+ ISplitReader reader=entry.getValue();
+ ReaderRunner runner=new ReaderRunner(split,reader);
+ workThreads.execute(runner);
+ }
+ try {
+ long sleepTime=this.pullIntervalMs-(System.currentTimeMillis()-start);
+ if(sleepTime>0){
+ Thread.sleep(sleepTime);
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public Map<String, ISplit> getAllSplitMap() {
+ List<ISplit> splits = fetchAllSplits();
+ if (splits == null) {
+ return new HashMap<>();
+ }
+ Map<String, ISplit> splitMap = new HashMap<>();
+ for (ISplit split : splits) {
+ splitMap.put(split.getQueueId(), split);
+ }
+ return splitMap;
+ }
+
+ protected void doSplitChanged(SplitChanged splitChanged) {
+ if (splitChanged == null) {
+ return;
+ }
+ if (splitChanged.getSplitCount() == 0) {
+ return;
+ }
+ if (splitChanged.isNewSplit()) {
+ doSplitAddition(splitChanged.getChangedSplits());
+ } else {
+ doSplitRelease(splitChanged.getChangedSplits());
+ }
+ }
+
+ protected void doSplitAddition(List<ISplit> changedSplits) {
+ if (changedSplits == null) {
+ return;
+ }
+ Set<String> splitIds = new HashSet<>();
+ for (ISplit split : changedSplits) {
+ splitIds.add(split.getQueueId());
+ }
+ addNewSplit(splitIds);
+ for (ISplit split : changedSplits) {
+ ISplitReader reader = createSplitReader(split);
+ reader.open(split);
+ reader.seek(loadSplitOffset(split));
+ splitReaders.put(split.getQueueId(), reader);
+ this.ownerSplits.put(split.getQueueId(), split);
+// logger.info("start next");
+// Thread thread = new Thread(new Runnable() {
+//
+// thread.setName("reader-task-" + reader.getSplit().getQueueId());
+// thread.start();
+ }
+
+ }
+
+ @Override
+ public String loadSplitOffset(ISplit split) {
+ String offset = null;
+ CheckPoint<String> checkPoint = checkPointManager.recover(this, split);
+ if (checkPoint != null) {
+ offset = JSON.parseObject(checkPoint.getData()).getString("offset");
+ }
+ return offset;
+ }
+
+ protected abstract ISplitReader createSplitReader(ISplit split);
+
+ protected void doSplitRelease(List<ISplit> changedSplits) {
+ boolean success = balance.getRemoveSplitLock();
+ if (!success) {
+ return;
+ }
+ try {
+ List<SplitCloseFuture> closeFutures = new ArrayList<>();
+ for (ISplit split : changedSplits) {
+ ISplitReader reader = this.splitReaders.get(split.getQueueId());
+ if (reader == null) {
+ continue;
+ }
+ SplitCloseFuture future = reader.close();
+ closeFutures.add(future);
+ }
+ for (SplitCloseFuture future : closeFutures) {
+ try {
+ if(!future.isDone()){
+ future.get();
+ }
+ this.splitReaders.remove(future.getSplit().getQueueId());
+ this.ownerSplits.remove(future.getSplit().getQueueId());
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (ExecutionException e) {
+ e.printStackTrace();
+ }
+ }
+
+ } finally {
+ balance.unLockRemoveSplitLock();
+ }
+
+ }
+
+
+ protected class ReaderRunner implements Runnable{
+ long mLastCheckTime = System.currentTimeMillis();
+ protected ISplit split;
+ protected ISplitReader reader;
+
+ public ReaderRunner(ISplit split,ISplitReader reader){
+ this.split=split;
+ this.reader=reader;
+ }
+
+ @Override
+ public void run() {
+ logger.info("start running");
+ if (reader.isInterrupt() == false) {
+ if (reader.next()) {
+ List<PullMessage> messages = reader.getMessage();
+ if (messages != null) {
+ for (PullMessage pullMessage : messages) {
+ String queueId = split.getQueueId();
+ String offset = pullMessage.getOffsetStr();
+ JSONObject msg = createJson(pullMessage.getMessage());
+ Message message = createMessage(msg, queueId, offset, false);
+ message.getHeader().setOffsetIsLong(pullMessage.getMessageOffset().isLongOfMainOffset());
+ executeMessage(message);
+ }
+ }
+ reader.notifyAll();
+ }
+ long curTime = System.currentTimeMillis();
+ if (curTime - mLastCheckTime > getCheckpointTime()) {
+ sendCheckpoint(reader.getSplit().getQueueId());
+ mLastCheckTime = curTime;
+ }
+
+
+ }else {
+ Set<String> removeSplits = new HashSet<>();
+ removeSplits.add(reader.getSplit().getQueueId());
+ removeSplit(removeSplits);
+ balance.unlockSplit(split);
+ reader.close();
+ synchronized (reader) {
+ reader.notifyAll();
+ }
+ }
+
+ }
+
+ }
+
+ @Override
+ public boolean supportNewSplitFind() {
+ return true;
+ }
+
+ @Override
+ public boolean supportRemoveSplitFind() {
+ return true;
+ }
+
+ @Override
+ public boolean supportOffsetRest() {
+ return true;
+ }
+
+ @Override
+ public Long getPullIntervalMs() {
+ return pullIntervalMs;
+ }
+
+ public String getBalanceName() {
+ return balanceName;
+ }
+
+ public void setBalanceName(String balanceName) {
+ this.balanceName = balanceName;
+ }
+
+ public int getBalanceTimeSecond() {
+ return balanceTimeSecond;
+ }
+
+ public void setBalanceTimeSecond(int balanceTimeSecond) {
+ this.balanceTimeSecond = balanceTimeSecond;
+ }
+
+ public void setPullIntervalMs(long pullIntervalMs) {
+ this.pullIntervalMs = pullIntervalMs;
+ }
+
+ @Override
+ public List<ISplit> ownerSplits() {
+ return new ArrayList(ownerSplits.values());
+ }
+
+}
\ No newline at end of file
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/CycleDynamicMultipleDBScanSource.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/CycleDynamicMultipleDBScanSource.java
new file mode 100644
index 00000000..561b48f2
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/CycleDynamicMultipleDBScanSource.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.channel.source.AbstractSource;
+import org.apache.rocketmq.streams.common.channel.source.ISource;
+import org.apache.rocketmq.streams.common.channel.source.systemmsg.ChangeTableNameMessage;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.context.Message;
+import org.apache.rocketmq.streams.common.metadata.MetaDataUtils;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.ThreadUtil;
+import org.apache.rocketmq.streams.connectors.IBoundedSource;
+import org.apache.rocketmq.streams.connectors.model.ReaderStatus;
+import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
+import org.apache.rocketmq.streams.connectors.source.filter.CycleSchedule;
+import org.apache.rocketmq.streams.connectors.source.filter.CycleScheduleFilter;
+import org.apache.rocketmq.streams.db.CycleSplit;
+
+/**
+ * @description
+ */
+public class CycleDynamicMultipleDBScanSource extends DynamicMultipleDBScanSource implements IBoundedSource, Serializable {
+
+ private static final long serialVersionUID = 6840988298037061128L;
+ private static final Log logger = LogFactory.getLog(CycleDynamicMultipleDBScanSource.class);
+
+ Map<String, Boolean> initReaderMap = new ConcurrentHashMap<>();
+ CycleSchedule.Cycle cycle;
+ transient AtomicInteger size = new AtomicInteger(0);
+
+ public CycleDynamicMultipleDBScanSource() {
+ super();
+ }
+
+ public CycleDynamicMultipleDBScanSource(CycleSchedule.Cycle cycle) {
+ super();
+ this.cycle = cycle;
+ }
+
+ public AtomicInteger getSize() {
+ return size;
+ }
+
+ public void setSize(AtomicInteger size) {
+ this.size = size;
+ }
+
+ /**
+ * @return
+ */
+ //todo
+ @Override
+ public synchronized List<ISplit> fetchAllSplits() {
+
+ if (this.filter == null) {
+ filter = new CycleScheduleFilter(cycle.getAllPattern());
+ }
+
+ //如果还是当前周期, 已经完成全部分区的加载, 则不在加载
+ if (size.get() == cycle.getCycleCount()) {
+ return splits;
+ }
+ String sourceName = createKey(this);
+ List<String> tableNames = MetaDataUtils.listTableNameByPattern(url, userName, password, logicTableName + "%");
+
+ logger.info(String.format("load all logic table : %s", Arrays.toString(tableNames.toArray())));
+ Iterator<String> it = tableNames.iterator();
+ while (it.hasNext()) {
+ String s = it.next();
+ String suffix = s.replace(logicTableName + "_", "");
+ if (filter.filter(sourceName, logicTableName, suffix)) {
+ logger.info(String.format("filter add %s", s));
+ CycleSplit split = new CycleSplit();
+ split.setLogicTableName(logicTableName);
+ split.setSuffix(suffix);
+ split.setCyclePeriod(cycle.getCycleDateStr());
+ String splitId = split.getQueueId();
+ if (initReaderMap.get(splitId) == null) {
+ initReaderMap.put(splitId, false);
+ splits.add(split);
+ size.incrementAndGet();
+ }
+ } else {
+ logger.info(String.format("filter remove %s", s));
+ it.remove();
+ }
+ }
+
+ this.tableNames = tableNames;
+ return splits;
+ }
+
+ public Map<String, Boolean> getInitReaderMap() {
+ return initReaderMap;
+ }
+
+ public void setInitReaderMap(Map<String, Boolean> initReaderMap) {
+ this.initReaderMap = initReaderMap;
+ }
+
+ @Override
+ public void finish() {
+ super.finish();
+ for (Map.Entry<String, Boolean> entry : initReaderMap.entrySet()) {
+ String key = entry.getKey();
+ Boolean value = entry.getValue();
+ if (value == false) {
+ logger.error(String.format("split[%s] reader is not finish, exit with error. ", key));
+ }
+ }
+ this.initReaderMap.clear();
+ this.initReaderMap = null;
+ splits.clear();
+ splits = null;
+ }
+
+ @Override
+ public boolean isFinished() {
+ List<ReaderStatus> readerStatuses = ReaderStatus.queryReaderStatusListBySourceName(createKey(this));
+ if (readerStatuses == null) {
+ return false;
+ }
+ return readerStatuses.size() == size.get();
+ }
+
+ @Override
+ protected ISplitReader createSplitReader(ISplit iSplit) {
+ return super.createSplitReader(iSplit);
+ }
+
+ private void sendChangeTableNameMessage() {
+ logger.info(String.format("start send change table name message."));
+ ChangeTableNameMessage changeTableNameMessage = new ChangeTableNameMessage();
+ changeTableNameMessage.setScheduleCycle(cycle.getCycleDateStr());
+ Message message = createMessage(new JSONObject(), null, null, false);
+ message.setSystemMessage(changeTableNameMessage);
+ message.getHeader().setSystemMessage(true);
+ executeMessage(message);
+ logger.info(String.format("finish send change table name message."));
+ }
+
+ @Override
+ public synchronized void boundedFinishedCallBack(ISplit iSplit) {
+ this.initReaderMap.put(iSplit.getQueueId(), true);
+ logger.info(String.format("current map is %s, key is %s. ", initReaderMap, iSplit.getQueueId()));
+ if (statusCheckerStart.compareAndSet(false, true)) {
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ while (!isFinished()) {
+ ThreadUtil.sleep(3 * 1000);
+ }
+ logger.info(String.format("source will be closed."));
+ sendChangeTableNameMessage(); //下发修改name的消息
+ ThreadUtil.sleep(1 * 1000);
+ finish();
+ }
+
+ });
+ thread.setName(createKey(this) + "_callback");
+ thread.start();
+ }
+ }
+
+ public CycleSchedule.Cycle getCycle() {
+ return cycle;
+ }
+
+ public void setCycle(CycleSchedule.Cycle cycle) {
+ this.cycle = cycle;
+ }
+
+ @Override
+ public String createCheckPointName() {
+ return super.createCheckPointName();
+ }
+
+ public synchronized int getTotalReader() {
+ return size.get();
+ }
+
+ public static String createKey(ISource iSource) {
+ AbstractSource source = (AbstractSource) iSource;
+ CycleSchedule.Cycle cycle = ((CycleDynamicMultipleDBScanSource) iSource).getCycle();
+ return MapKeyUtil.createKey(source.getNameSpace(), source.getGroupName(), source.getConfigureName(), source.getTopic(), cycle.getCycleDateStr());
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/DynamicMultipleDBScanSource.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/DynamicMultipleDBScanSource.java
new file mode 100644
index 00000000..ea2a118b
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/DynamicMultipleDBScanSource.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.metadata.MetaDataUtils;
+import org.apache.rocketmq.streams.connectors.reader.DBScanReader;
+import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
+import org.apache.rocketmq.streams.connectors.source.filter.DataFormatPatternFilter;
+import org.apache.rocketmq.streams.connectors.source.filter.PatternFilter;
+import org.apache.rocketmq.streams.db.DynamicMultipleDBSplit;
+
+/**
+ * @description DynamicMultipleDBScanSource
+ */
+public class DynamicMultipleDBScanSource extends AbstractPullSource implements Serializable {
+
+ private static final long serialVersionUID = 3987103552547019739L;
+ private static final Log logger = LogFactory.getLog(DynamicMultipleDBScanSource.class);
+ public static final int DEFAULT_BATCH_SIZE = 50;
+ public static final int MAX_BATCH_SIZE = 100;
+
+ String url;
+ String userName;
+ String password;
+ String logicTableName;
+ String suffix;
+ int batchSize;
+ List<String> tableNames;
+ List<ISplit> splits;
+ transient volatile AtomicBoolean statusCheckerStart = new AtomicBoolean(false);
+
+ //todo
+ transient PatternFilter filter;
+
+ public DynamicMultipleDBScanSource() {
+ splits = new ArrayList<>();
+ }
+
+ @Override
+ protected boolean initConfigurable() {
+ setTopic(logicTableName);
+ return super.initConfigurable();
+ }
+
+ @Override
+ protected boolean isNotDataSplit(String queueId) {
+ return tableNames.contains(queueId);
+ }
+
+ @Override
+ protected ISplitReader createSplitReader(ISplit split) {
+
+ DBScanReader reader = new DBScanReader();
+ reader.setISplit(split);
+ reader.setUrl(url);
+ reader.setUserName(userName);
+ reader.setPassword(password);
+ reader.setTableName(String.valueOf(split.getQueue()));
+ int local = batchSize <= 0 ? DEFAULT_BATCH_SIZE : batchSize;
+ local = local > MAX_BATCH_SIZE ? MAX_BATCH_SIZE : local;
+ reader.setBatchSize(local);
+ reader.setISource(this);
+ logger.info(String.format("create reader for split %s", split.getQueueId()));
+ return reader;
+ }
+
+ @Override
+ public List<ISplit> fetchAllSplits() {
+
+ if (filter == null) {
+ filter = new DataFormatPatternFilter();
+ }
+
+// String sourceName = createKey(this);
+
+ tableNames = MetaDataUtils.listTableNameByPattern(url, userName, password, logicTableName + "%");
+
+ logger.info(String.format("load all logic table : %s", Arrays.toString(tableNames.toArray())));
+
+ for (String s : tableNames) {
+ String suffix = s.replace(logicTableName + "_", "");
+ if (filter.filter(null, logicTableName, suffix)) {
+ logger.info(String.format("filter add %s", s));
+ DynamicMultipleDBSplit split = new DynamicMultipleDBSplit();
+ split.setLogicTableName(logicTableName);
+ split.setSuffix(suffix);
+ splits.add(split);
+ } else {
+ logger.info(String.format("filter remove %s", s));
+ }
+
+ }
+ return splits;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getLogicTableName() {
+ return logicTableName;
+ }
+
+ public void setLogicTableName(String logicTableName) {
+ this.logicTableName = logicTableName;
+ }
+
+ public String getSuffix() {
+ return suffix;
+ }
+
+ public void setSuffix(String suffix) {
+ this.suffix = suffix;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public List<String> getTableNames() {
+ return tableNames;
+ }
+
+ public void setTableNames(List<String> tableNames) {
+ this.tableNames = tableNames;
+ }
+
+ public List<ISplit> getSplits() {
+ return splits;
+ }
+
+ public void setSplits(List<ISplit> splits) {
+ this.splits = splits;
+ }
+
+ public PatternFilter getFilter() {
+ return filter;
+ }
+
+ public void setFilter(PatternFilter filter) {
+ this.filter = filter;
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/IPullSource.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/IPullSource.java
new file mode 100644
index 00000000..6733911d
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/IPullSource.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.streams.common.channel.source.ISource;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+
+/**
+ * poll message，need balance
+ */
+public interface IPullSource<T extends ISource> extends ISource<T> {
+
+ /**
+ * 拥有的分片格式
+ *
+ * @return
+ */
+ Collection<ISplit> ownerSplits();
+
+ /**
+ * get all split for the source
+ *
+ * @return
+ */
+ List<ISplit> fetchAllSplits();
+
+ /**
+ * get all split for the source
+ *
+ * @return
+ */
+ Map<String, ISplit> getAllSplitMap();
+
+ Long getPullIntervalMs();
+
+ /**
+ * get cusor from store
+ *
+ * @return
+ */
+ String loadSplitOffset(ISplit split);
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/MutilBatchTaskSource.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/MutilBatchTaskSource.java
new file mode 100644
index 00000000..d82ba32a
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/MutilBatchTaskSource.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source;
+
+import com.alibaba.fastjson.JSON;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.checkpoint.CheckPoint;
+import org.apache.rocketmq.streams.common.topology.ChainPipeline;
+import org.apache.rocketmq.streams.common.topology.model.Pipeline;
+import org.apache.rocketmq.streams.common.topology.task.TaskAssigner;
+import org.apache.rocketmq.streams.common.utils.DipperThreadLocalUtil;
+import org.apache.rocketmq.streams.common.utils.RuntimeUtil;
+import org.apache.rocketmq.streams.connectors.model.PullMessage;
+import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
+import org.apache.rocketmq.streams.connectors.reader.SplitCloseFuture;
+
+public class MutilBatchTaskSource extends AbstractPullSource {
+
+ @Override protected ISplitReader createSplitReader(ISplit split) {
+ return new ISplitReader() {
+ protected transient ISplit split;
+ protected boolean isInterrupt;
+ protected boolean isClose;
+ protected transient AtomicLong offsetGenerator=new AtomicLong(1000000000);
+ @Override public void open(ISplit split) {
+ this.split=split;
+ }
+
+ @Override public boolean next() {
+ return true;
+ }
+
+ @Override public List<PullMessage> getMessage() {
+ PipelineSplit pipelineSplit=(PipelineSplit)split;
+ pipelineSplit.getQueue().startChannel();
+ return null;
+ }
+
+ @Override public SplitCloseFuture close() {
+ isClose=true;
+ return new SplitCloseFuture(this,split);
+ }
+
+ @Override public void seek(String cursor) {
+
+ }
+
+ @Override public String getProgress() {
+ return RuntimeUtil.getDipperInstanceId()+"_"+offsetGenerator.incrementAndGet();
+ }
+
+ @Override public long getDelay() {
+ return 0;
+ }
+
+ @Override public long getFetchedDelay() {
+ return getPullIntervalMs();
+ }
+
+ @Override public boolean isClose() {
+ return isClose;
+ }
+
+ @Override public ISplit getSplit() {
+ return split;
+ }
+
+ @Override public boolean isInterrupt() {
+ return isInterrupt;
+ }
+
+ @Override public boolean interrupt() {
+ isInterrupt=true;
+ return isInterrupt;
+ }
+ };
+ }
+
+ @Override protected boolean isNotDataSplit(String queueId) {
+ return false;
+ }
+
+ @Override public List<ISplit> fetchAllSplits() {
+
+ List<TaskAssigner> taskAssigners = configurableService.queryConfigurableByType(TaskAssigner.TYPE);
+ if (taskAssigners == null) {
+ return null;
+ }
+ String taskName = getConfigureName();
+ List<ISplit> splits=new ArrayList<>();
+ for (TaskAssigner taskAssigner : taskAssigners) {
+ if (!taskName.equals(taskAssigner.getTaskName())) {
+ continue;
+ }
+ String pipelineName = taskAssigner.getPipelineName();
+ if(pipelineName!=null){
+ ChainPipeline<?> pipeline = configurableService.queryConfigurable(Pipeline.TYPE, pipelineName);
+ if (pipeline != null) {
+ splits.add(new PipelineSplit(pipeline));
+ }
+ }
+ }
+ return splits;
+ }
+
+
+ protected class PipelineSplit implements ISplit<PipelineSplit, ChainPipeline>{
+ protected ChainPipeline chainPipeline;
+ public PipelineSplit(ChainPipeline chainPipeline){
+ this.chainPipeline=chainPipeline;
+ }
+
+ @Override public String getQueueId() {
+ return chainPipeline.getConfigureName();
+ }
+
+ @Override public ChainPipeline getQueue() {
+ return chainPipeline;
+ }
+
+ @Override public int compareTo(PipelineSplit o) {
+ return chainPipeline.getConfigureName().compareTo(o.getQueueId());
+ }
+
+ @Override public String toJson() {
+ return chainPipeline.toJson();
+ }
+
+ @Override public void toObject(String jsonString) {
+ ChainPipeline pipeline=new ChainPipeline();
+ pipeline.toObject(jsonString);
+ this.chainPipeline=pipeline;
+ }
+ }
+
+ @Override
+ public String loadSplitOffset(ISplit split) {
+ return null;
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/SourceInstance.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/SourceInstance.java
new file mode 100644
index 00000000..c0da5b6e
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/SourceInstance.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source;
+
+/**
+ * i个消息队列的实例，一个实例i个
+ */
+public class SourceInstance {
+ protected String sourceInstanceId;
+
+
+ public SourceInstance(String sourceInstanceId){
+ this.sourceInstanceId=sourceInstanceId;
+ }
+
+ public String getSourceInstanceId() {
+ return sourceInstanceId;
+ }
+
+ public void setSourceInstanceId(String sourceInstanceId) {
+ this.sourceInstanceId = sourceInstanceId;
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/AbstractPatternFilter.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/AbstractPatternFilter.java
new file mode 100644
index 00000000..0d5368a7
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/AbstractPatternFilter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.io.Serializable;
+
+/**
+ * @description
+ */
+public abstract class AbstractPatternFilter implements PatternFilter, Serializable {
+
+ private static final long serialVersionUID = 6500945777421871431L;
+
+ PatternFilter next;
+
+ public abstract boolean filter(String sourceName, String logicTableName, String tableName);
+
+
+ @Override
+ public PatternFilter setNext(PatternFilter filter) {
+ this.next = filter;
+ return this;
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/BoundedPatternFilter.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/BoundedPatternFilter.java
new file mode 100644
index 00000000..c06de98d
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/BoundedPatternFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.io.Serializable;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.connectors.model.ReaderStatus;
+
+/**
+ * @description 过滤掉已经完成的reader
+ */
+@Deprecated
+public class BoundedPatternFilter extends AbstractPatternFilter implements Serializable {
+
+ static final Log logger = LogFactory.getLog(BoundedPatternFilter.class);
+
+ @Override
+ public boolean filter(String sourceName, String logicTableName, String tableName) {
+
+ ReaderStatus readerStatus = ReaderStatus.queryReaderStatusByUK(sourceName, logicTableName + "_" + tableName);
+ if (readerStatus != null) {
+ logger.info(String.format("filter sourceName %s, logicTableName %s, suffix %s. ", sourceName, logicTableName, tableName));
+ logger.info(String.format("query result %s", readerStatus.toString()));
+ return true;
+ }
+ if (next == null) {
+ return false;
+ }
+ return next.filter(sourceName, logicTableName, tableName);
+ }
+
+ @Override
+ public PatternFilter setNext(PatternFilter filter) {
+ super.setNext(filter);
+ return this;
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePatternFilter.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePatternFilter.java
new file mode 100644
index 00000000..3a0193f3
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePatternFilter.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * @description 用来做分区选取
+ */
+public class CyclePatternFilter extends AbstractPatternFilter implements Serializable {
+
+ private static final long serialVersionUID = -5151597286296228754L;
+
+ public static final int INIT_CYCLE_VERSION = 0;
+
+ CyclePeriod cyclePeriod;
+
+ Date curCycleDateTime; //当前调度周期时间
+
+ long cycleId;
+
+ String firstStartTime; //当前最小时间
+
+ List<String> allPatterns;
+
+ String expression;
+
+ boolean isInit;
+
+ //历史数据读取时使用,表示比起当前相差多少个调度周期
+ final long cycleDiff;
+
+ //todo expr解析
+ public CyclePatternFilter(String expr, Date date) throws ParseException {
+ expression = expr;
+ cycleId = INIT_CYCLE_VERSION;
+ cyclePeriod = CyclePeriod.getInstance(expression);
+ curCycleDateTime = calCycleDateTime(date);
+ allPatterns = new ArrayList<>();
+ isInit = true;
+ if(cyclePeriod.isHistory){
+ Date tmp = cyclePeriod.getHisDate();
+ cycleDiff = curCycleDateTime.getTime()/1000 * 1000 - tmp.getTime()/1000*1000;
+ }else{
+ cycleDiff = 0;
+ }
+ }
+
+
+ /**
+ *
+ * @return 返回date格式的调度周期时间
+ */
+ private Date calCycleDateTime(Date date){
+ return cyclePeriod.format(date);
+ }
+
+ private long calCycle(Date date){
+ Date tmp = calCycleDateTime(date);
+ if(tmp.getTime()/1000 == curCycleDateTime.getTime()/1000){
+ return cycleId;
+ }
+ return nextCycle(tmp);
+ }
+
+ private long nextCycle(Date date){
+ curCycleDateTime = date;
+ cycleId++;
+ calAllPattern();
+ return cycleId;
+ }
+
+ private void calAllPattern(){
+ allPatterns.clear();
+ for(int i = 1; i <= cyclePeriod.getCycle(); i++){
+ long d = (curCycleDateTime.getTime()/1000)*1000 - i * cyclePeriod.getInterval() - cycleDiff;
+ String s = cyclePeriod.getDateFormat().format(new Date(d));
+ allPatterns.add(s);
+ }
+ firstStartTime = allPatterns.get(allPatterns.size() - 1);
+ }
+
+ public boolean isNextCycle(Date date){
+ if(isInit){
+ isInit = false;
+ calAllPattern();
+ return true;
+ }
+ long tmp = cycleId;
+ return calCycle(date) > tmp;
+ }
+
+ public List<String> getAllPatterns() {
+ return allPatterns;
+ }
+
+ public long getCycleId() {
+ return cycleId;
+ }
+
+ public Date getCurCycleDateTime(){
+ return curCycleDateTime;
+ }
+
+ public String getCurCycleDateTimeStr(){
+ return cyclePeriod.getDateFormat().format(curCycleDateTime);
+ }
+
+ public long getCycleDiff() {
+ return cycleDiff;
+ }
+
+ public long getCyclePeriodDiff(){
+ return cycleDiff/cyclePeriod.getInterval();
+ }
+
+ public int getCycle(){
+ return cyclePeriod.getCycle();
+ }
+
+ public String getFirstStartTime() {
+ return firstStartTime;
+ }
+
+ @Override
+ public boolean filter(String sourceName, String logicTableName, String tableName) {
+ return allPatterns.contains(tableName);
+ }
+
+
+
+ public static void main(String[] args) throws ParseException {
+
+ CyclePatternFilter cycle = new CyclePatternFilter("yyyyMMddHHmm - 15m", new Date());
+ System.out.println(cycle);
+
+ System.out.println(cycle.filter(null, null, "202109131650"));
+ System.out.println(cycle.filter(null, null, "20210902000000"));
+ System.out.println(cycle.filter(null, null, "20210908000000"));
+ System.out.println(cycle.filter(null, null, "20210910000000"));
+ System.out.println(cycle.filter(null, null, "20210909230000"));
+
+ System.out.println(new SimpleDateFormat("yyyyMMddHH").parse("2021090923"));
+ System.out.println(new SimpleDateFormat("yyyyMMddhhmmss").parse("20210909230000"));
+ System.out.println(new SimpleDateFormat("yyyyMMddHHmmss").parse("20210909100000"));
+ System.out.println(new SimpleDateFormat("yyyyMMddhhmmss").parse("20210909100000"));
+
+
+
+
+ }
+
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePeriod.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePeriod.java
new file mode 100644
index 00000000..4e6cdd6a
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePeriod.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * @Description
+ */
+public enum CyclePeriod {
+
+ CYCLE_PERIOD_DATE() {
+ @Override
+ void argsParser(String expr) throws ParseException {
+ super.argsParser(expr);
+ interval = 24 * 3600 * 1000;
+ int length = expr.length();
+ if (length == 8 && checkFormat(expr, PatternFilter.yyyyMMdd)) {
+ format = PatternFilter.yyyyMMdd;
+ } else if (length == 14 && checkFormat(expr, PatternFilter.yyyyMMddHHmmss)) {
+ format = PatternFilter.yyyyMMddHHmmss;
+ } else {
+ throw new RuntimeException(String.format("unsupported format : %s, only support yyyymmdd 、 yyyymmddhhmmss.", expr));
+ }
+ }
+
+ @Override
+ Date format(Date strDate) {
+ Date date = new Date(strDate.getTime());
+ date.setHours(0);
+ date.setMinutes(0);
+ date.setSeconds(0);
+ return date;
+ }
+
+ },
+ CYCLE_PERIOD_HOUR() {
+ @Override
+ void argsParser(String expr) throws ParseException {
+ super.argsParser(expr);
+ interval = 3600 * 1000;
+
+ int length = expr.length();
+ if (length == 10 && checkFormat(expr, PatternFilter.yyyyMMddHH)) {
+ format = PatternFilter.yyyyMMddHH;
+ } else if (length == 14 && checkFormat(expr, PatternFilter.yyyyMMddHHmmss)) {
+ format = PatternFilter.yyyyMMddHHmmss;
+ } else {
+ throw new RuntimeException(String.format("unsupported format : %s, only support yyyymmdd 、 yyyymmddhhmmss.", expr));
+ }
+ }
+
+ @Override
+ Date format(Date strDate) {
+ Date date = new Date(strDate.getTime());
+ date.setMinutes(0);
+ date.setSeconds(0);
+ return date;
+ }
+
+ },
+ CYCLE_PERIOD_MINUTE() {
+ @Override
+ void argsParser(String expr) throws ParseException {
+ super.argsParser(expr);
+ interval = 60 * 1000;
+ int length = expr.length();
+ if (length == 12 && checkFormat(expr, PatternFilter.yyyyMMddHHmm)) {
+ format = PatternFilter.yyyyMMddHHmm;
+ } else if (length == 14 && checkFormat(expr, PatternFilter.yyyyMMddHHmmss)) {
+ format = PatternFilter.yyyyMMddHHmmss;
+ } else {
+ throw new RuntimeException(String.format("unsupported format : %s, only support yyyymmdd 、 yyyymmddhhmmss.", expr));
+ }
+ }
+
+ @Override
+ Date format(Date strDate) {
+ Date date = new Date(strDate.getTime());
+ date.setSeconds(0);
+ return date;
+ }
+
+ };
+
+ boolean isHistory = false;
+
+ long interval;
+
+ int cycle;
+
+ String format;
+
+ String hisDateString;
+
+ static final Log logger = LogFactory.getLog(CyclePeriod.class);
+
+ void argsParser(String expr) throws ParseException {
+ if (expr.matches("^\\d+$")) {
+ isHistory = true;
+ hisDateString = expr;
+ }
+ }
+
+ Date format(Date strDate) {
+ throw new RuntimeException(String.format("unsupported type.", strDate));
+ }
+
+ /**
+ * expr可能是yyyymmdd 或者 20210917
+ *
+ * @param expr
+ * @param format
+ * @return
+ */
+ final boolean checkFormat(String expr, String format) {
+
+ if (!isHistory) {
+ return expr.equalsIgnoreCase(format);
+ }
+
+ try {
+ new SimpleDateFormat(format).parse(expr);
+ return true;
+ } catch (ParseException e) {
+ logger.error(String.format("error format, expr is %s, format is %s.", expr, format));
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ public Date getHisDate() throws ParseException {
+ return getDateFormat().parse(hisDateString);
+ }
+
+ public SimpleDateFormat getDateFormat() {
+ return new SimpleDateFormat(format);
+ }
+
+ public long getInterval() {
+ return interval;
+ }
+
+ public boolean isHistory() {
+ return isHistory;
+ }
+
+ public void setHistory(boolean history) {
+ isHistory = history;
+ }
+
+ public void setInterval(long interval) {
+ this.interval = interval;
+ }
+
+ public int getCycle() {
+ return cycle;
+ }
+
+ public void setCycle(int cycle) {
+ this.cycle = cycle;
+ }
+
+ public void setFormat(String format) {
+ this.format = format;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public String getHisDateString() {
+ return hisDateString;
+ }
+
+ public void setHisDateString(String hisDateString) {
+ this.hisDateString = hisDateString;
+ }
+
+ public static CyclePeriod getInstance(String expression) throws ParseException {
+
+ String[] str = expression.split("\\-");
+ assert str.length == 2 : String.format("expression error : %s. ", expression);
+ String expr = str[0].trim();
+ String tmp = str[1].trim().toLowerCase();
+ String cycleStr = tmp.substring(0, tmp.length() - 1);
+ int cycle = Integer.parseInt(cycleStr);
+ CyclePeriod cyclePeriod = null;
+ if (tmp.endsWith("d")) {
+ cyclePeriod = CYCLE_PERIOD_DATE;
+ } else if (tmp.endsWith("h")) {
+ cyclePeriod = CYCLE_PERIOD_HOUR;
+ } else if (tmp.endsWith("m")) {
+ cyclePeriod = CYCLE_PERIOD_MINUTE;
+ } else {
+ new RuntimeException(String.format("unsupported format : %s", expression));
+ }
+ cyclePeriod.argsParser(expr);
+ cyclePeriod.cycle = cycle;
+
+ return cyclePeriod;
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleSchedule.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleSchedule.java
new file mode 100644
index 00000000..ba9a2797
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleSchedule.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+
+/**
+ * @description 用来做分区选取
+ */
+public class CycleSchedule implements Serializable {
+
+ private static final long serialVersionUID = -5151597286296228754L;
+ public static final int INIT_CYCLE_VERSION = 0;
+ private static CycleSchedule INSTANCE;
+ CyclePeriod cyclePeriod;
+ AtomicLong cycleId = new AtomicLong(0);
+ String expression;
+ boolean isInit;
+ //历史数据读取时使用,表示比起当前相差多少个调度周期
+ final long cycleDiff;
+
+ public CycleSchedule(String expr, Date date) throws ParseException {
+ Date local = subMs(date);
+ expression = expr;
+ cycleId.set(INIT_CYCLE_VERSION);
+ cyclePeriod = CyclePeriod.getInstance(expression);
+ isInit = true;
+ if (cyclePeriod.isHistory) {
+ Date curCycleDateTime = calCycleDateTime(local);
+ Date tmp = subMs(cyclePeriod.getHisDate());
+ cycleDiff = curCycleDateTime.getTime() - tmp.getTime();
+ } else {
+ cycleDiff = 0;
+ }
+ }
+
+ /**
+ * 去掉毫秒时间戳
+ *
+ * @param date
+ * @return
+ */
+ private Date subMs(Date date) {
+ long time = date.getTime() / 1000 * 1000;
+ return new Date(time);
+ }
+
+ /**
+ * @return 返回date格式的调度周期时间
+ */
+ private Date calCycleDateTime(Date date) {
+ return cyclePeriod.format(date);
+ }
+
+ public Cycle nextCycle(Date date) {
+ Date local = subMs(date);
+ local = cyclePeriod.format(local);
+ if (isInit) {
+ isInit = false;
+ } else {
+ cycleId.incrementAndGet();
+ }
+ List<String> ret = calAllPattern(local);
+ Cycle cycle = new Cycle();
+ cycle.setCycleId(cycleId.get());
+ cycle.setAllPattern(ret);
+ cycle.setCycleDateStr(calCycleDateStr(local));
+ cycle.setCycleCount(cyclePeriod.getCycle());
+ cycle.setCurDateStr(cyclePeriod.getDateFormat().format(local));
+ cycle.setCycleDiff(cycleDiff);
+ return cycle;
+ }
+
+ private String calCycleDateStr(Date date) {
+ long d = date.getTime() - cycleDiff;
+ Date d1 = new Date(d);
+ return cyclePeriod.getDateFormat().format(d1);
+ }
+
+ private List<String> calAllPattern(Date date) {
+ List<String> allPatterns = new ArrayList<>();
+ for (int i = 1; i <= cyclePeriod.getCycle(); i++) {
+ long d = date.getTime() - i * cyclePeriod.getInterval() - cycleDiff;
+ String s = cyclePeriod.getDateFormat().format(new Date(d));
+ allPatterns.add(s);
+ }
+ return allPatterns;
+ }
+
+ public CyclePeriod getCyclePeriod() {
+ return cyclePeriod;
+ }
+
+ public void setCyclePeriod(CyclePeriod cyclePeriod) {
+ this.cyclePeriod = cyclePeriod;
+ }
+
+ public AtomicLong getCycleId() {
+ return cycleId;
+ }
+
+ public void setCycleId(AtomicLong cycleId) {
+ this.cycleId = cycleId;
+ }
+
+ public String getExpression() {
+ return expression;
+ }
+
+ public void setExpression(String expression) {
+ this.expression = expression;
+ }
+
+ public boolean isInit() {
+ return isInit;
+ }
+
+ public void setInit(boolean init) {
+ isInit = init;
+ }
+
+ public long getCycleDiff() {
+ return cycleDiff;
+ }
+
+ public static CycleSchedule getInstance(String expr, Date date) {
+ if (INSTANCE == null) {
+ synchronized (CycleSchedule.class) {
+ if (INSTANCE == null) {
+ try {
+ INSTANCE = new CycleSchedule(expr, date);
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ return INSTANCE;
+ }
+
+ public static class Cycle extends BasedConfigurable implements Serializable {
+
+ private static final long serialVersionUID = 4842560538716388622L;
+ Long cycleId;
+ List<String> allPattern;
+ String cycleDateStr;
+ Integer cycleCount;
+ String curDateStr;
+ long cycleDiff;
+
+ public Integer getCycleCount() {
+ return cycleCount;
+ }
+
+ public void setCycleCount(Integer cycleCount) {
+ this.cycleCount = cycleCount;
+ }
+
+ public Cycle() {
+ }
+
+ public Long getCycleId() {
+ return cycleId;
+ }
+
+ public void setCycleId(Long cycleId) {
+ this.cycleId = cycleId;
+ }
+
+ public List<String> getAllPattern() {
+ return allPattern;
+ }
+
+ public void setAllPattern(List<String> allPattern) {
+ this.allPattern = allPattern;
+ }
+
+ public String getCycleDateStr() {
+ return cycleDateStr;
+ }
+
+ public void setCycleDateStr(String cycleDateStr) {
+ this.cycleDateStr = cycleDateStr;
+ }
+
+ public String getCurDateStr() {
+ return curDateStr;
+ }
+
+ public void setCurDateStr(String curDateStr) {
+ this.curDateStr = curDateStr;
+ }
+
+ public long getCycleDiff() {
+ return cycleDiff;
+ }
+
+ public void setCycleDiff(long cycleDiff) {
+ this.cycleDiff = cycleDiff;
+ }
+
+ @Override
+ public String toString() {
+ return "Cycle{" +
+ "cycleId=" + cycleId +
+ ", cycleDateStr='" + cycleDateStr + '\'' +
+ ", cycleCount=" + cycleCount +
+ ", curDateStr='" + curDateStr + '\'' +
+ ", cycleDiff=" + cycleDiff +
+ ", allPattern=" + Arrays.toString(allPattern.toArray()) +
+ '}';
+ }
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleScheduleFilter.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleScheduleFilter.java
new file mode 100644
index 00000000..507739d8
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleScheduleFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * @description
+ */
+public class CycleScheduleFilter extends AbstractPatternFilter implements Serializable {
+
+ List<String> allPattern;
+
+ public CycleScheduleFilter(List<String> allPattern){
+ this.allPattern = allPattern;
+ }
+
+ @Override
+ public boolean filter(String sourceName, String logicTableName, String tableName) {
+ return allPattern.contains(tableName);
+ }
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/DataFormatPatternFilter.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/DataFormatPatternFilter.java
new file mode 100644
index 00000000..0cdc0762
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/DataFormatPatternFilter.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * @description
+ */
+public class DataFormatPatternFilter extends AbstractPatternFilter implements Serializable {
+
+ private static final long serialVersionUID = 3604787588465242642L;
+
+ static final Log logger = LogFactory.getLog(DataFormatPatternFilter.class);
+
+ static final String yyyyMMddHHmmss = "yyyyMMddHHmmss";
+ static final String yyyyMMdd = "yyyyMMdd";
+ static final String yyyyMMddHH = "yyyyMMddHH";
+
+ SimpleDateFormat format1 = new SimpleDateFormat(yyyyMMdd);
+ SimpleDateFormat format2 = new SimpleDateFormat(yyyyMMddHH);
+ SimpleDateFormat format3 = new SimpleDateFormat(yyyyMMddHHmmss);
+
+ @Override
+ public boolean filter(String sourceName, String logicTableName, String tableNameSuffix) {
+
+ int len = tableNameSuffix.length();
+ boolean isFilter = false;
+
+ switch (len) {
+ case 8:
+ try {
+ format1.parse(tableNameSuffix);
+ isFilter = true;
+ } catch (ParseException e) {
+ e.printStackTrace();
+ isFilter = false;
+ }
+ break;
+ case 10:
+ try {
+ format2.parse(tableNameSuffix);
+ isFilter = true;
+ } catch (ParseException e) {
+ e.printStackTrace();
+ isFilter = false;
+ }
+ break;
+ case 14:
+ try {
+ format3.parse(tableNameSuffix);
+ isFilter = true;
+ } catch (ParseException e) {
+ e.printStackTrace();
+ isFilter = false;
+ }
+ break;
+ }
+
+ if (isFilter) {
+ logger.info(String.format("filter sourceName %s, logicTableName %s, suffix %s", sourceName, logicTableName, tableNameSuffix));
+ return true;
+ }
+ if (next != null) {
+ return next.filter(sourceName, logicTableName, tableNameSuffix);
+ }
+ return false;
+ }
+
+ @Override
+ public PatternFilter setNext(PatternFilter filter) {
+ super.setNext(filter);
+ return this;
+ }
+
+ public PatternFilter getNext() {
+ return next;
+ }
+
+ public static void main(String[] args) {
+ DataFormatPatternFilter filter = new DataFormatPatternFilter();
+// System.out.println(filter.filter("20200101"));
+// System.out.println(filter.filter("2020010101"));
+// System.out.println(filter.filter("20200101010101"));
+
+ }
+
+}
diff --git a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/PatternFilter.java b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/PatternFilter.java
new file mode 100644
index 00000000..42365007
--- /dev/null
+++ b/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/PatternFilter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.connectors.source.filter;
+
+/**
+ * @description
+ */
+public interface PatternFilter {
+
+ String yyyyMMddHHmmss = "yyyyMMddHHmmss";
+ String yyyyMMdd = "yyyyMMdd";
+ String yyyyMMddHH = "yyyyMMddHH";
+ String yyyyMMddHHmm = "yyyyMMddHHmm";
+
+
+ /**
+ * 根据sourceName和tableName判断是否符合
+ * @param sourceName
+ * @param tableName
+ * @return
+ */
+ boolean filter(String sourceName, String logicTableName, String tableName);
+
+ PatternFilter setNext(PatternFilter filter);
+
+
+}
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/offset/WindowMaxValueProcessor.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/offset/WindowMaxValueProcessor.java
index cf09bf48..4aa86ae4 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/offset/WindowMaxValueProcessor.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/offset/WindowMaxValueProcessor.java
@@ -127,7 +127,7 @@ public class WindowMaxValueProcessor {
}
String keyPrefix = MapKeyUtil.createKey(name, splitId);
- String sql="select * from "+ ORMUtil.getTableName(WindowMaxValue.class)+ " where msg_key like '"+keyPrefix+"%'";
+ String sql = "select * from " + ORMUtil.getTableName(WindowMaxValue.class) + " where configure_name like '%" + name + "%' and partition like '%" + splitId + "%'";
List<WindowMaxValue> windowMaxValues = ORMUtil.queryForList(sql, null, WindowMaxValue.class);
if (windowMaxValues == null || windowMaxValues.size() == 0) {
return result;
[rocketmq-streams] 10/16: Merge branch 'replaceDB' into devForEnterprise
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit f1d507c043872553fcc4841876f307ddc36d35c6
Merge: 5e91a37a ff92d26b
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 30 10:16:25 2022 +0800
Merge branch 'replaceDB' into devForEnterprise
.../streams/source/MessageListenerDelegator.java | 12 +++++-
.../rocketmq/streams/source/RocketMQSource.java | 44 ++++++++++++++--------
.../streams/examples/join/RocketmqJoinExample.java | 2 +-
.../streams/window/operator/AbstractWindow.java | 1 -
4 files changed, 40 insertions(+), 19 deletions(-)
[rocketmq-streams] 08/16: 1.0.2-preview-SNAPSHOT -> 1.0.3-preview-SNAPSHOT
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 5e91a37a7656a951052adb8618a41191ffae9341
Author: 维章 <un...@gmail.com>
AuthorDate: Fri May 27 11:32:11 2022 +0800
1.0.2-preview-SNAPSHOT -> 1.0.3-preview-SNAPSHOT
---
pom.xml | 2 +-
rocketmq-streams-channel-db/pom.xml | 2 +-
rocketmq-streams-channel-es/pom.xml | 2 +-
rocketmq-streams-channel-http/pom.xml | 2 +-
rocketmq-streams-channel-mqtt/pom.xml | 2 +-
rocketmq-streams-channel-rocketmq/pom.xml | 2 +-
rocketmq-streams-channel-syslog/pom.xml | 2 +-
rocketmq-streams-clients/pom.xml | 2 +-
rocketmq-streams-commons/pom.xml | 2 +-
rocketmq-streams-configurable/pom.xml | 2 +-
rocketmq-streams-db-operator/pom.xml | 2 +-
rocketmq-streams-examples/pom.xml | 2 +-
rocketmq-streams-filter/pom.xml | 2 +-
rocketmq-streams-script/pom.xml | 2 +-
rocketmq-streams-serviceloader/pom.xml | 2 +-
rocketmq-streams-state/pom.xml | 2 +-
rocketmq-streams-transport-minio/pom.xml | 2 +-
rocketmq-streams-window/pom.xml | 2 +-
18 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/pom.xml b/pom.xml
index 8bbc1651..dfb5a031 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
<name>Apache RocketMQ Streams ${project.version}</name>
<packaging>pom</packaging>
<url>https://rocketmq.apache.org/</url>
diff --git a/rocketmq-streams-channel-db/pom.xml b/rocketmq-streams-channel-db/pom.xml
index 82a66995..a215eeaf 100755
--- a/rocketmq-streams-channel-db/pom.xml
+++ b/rocketmq-streams-channel-db/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-channel-db</artifactId>
<name>ROCKETMQ STREAMS :: channel-db</name>
diff --git a/rocketmq-streams-channel-es/pom.xml b/rocketmq-streams-channel-es/pom.xml
index b4ce4d85..a21b7e72 100755
--- a/rocketmq-streams-channel-es/pom.xml
+++ b/rocketmq-streams-channel-es/pom.xml
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-channel-es</artifactId>
<name>ROCKETMQ STREAMS :: channel-es</name>
diff --git a/rocketmq-streams-channel-http/pom.xml b/rocketmq-streams-channel-http/pom.xml
index ac2cbc17..3741965b 100644
--- a/rocketmq-streams-channel-http/pom.xml
+++ b/rocketmq-streams-channel-http/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/rocketmq-streams-channel-mqtt/pom.xml b/rocketmq-streams-channel-mqtt/pom.xml
index 637f656d..b7c2922d 100644
--- a/rocketmq-streams-channel-mqtt/pom.xml
+++ b/rocketmq-streams-channel-mqtt/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>rocketmq-streams</artifactId>
<groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/rocketmq-streams-channel-rocketmq/pom.xml b/rocketmq-streams-channel-rocketmq/pom.xml
index 31f6d478..7c379b94 100644
--- a/rocketmq-streams-channel-rocketmq/pom.xml
+++ b/rocketmq-streams-channel-rocketmq/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>rocketmq-streams</artifactId>
<groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/rocketmq-streams-channel-syslog/pom.xml b/rocketmq-streams-channel-syslog/pom.xml
index 61e9aac6..5614429a 100755
--- a/rocketmq-streams-channel-syslog/pom.xml
+++ b/rocketmq-streams-channel-syslog/pom.xml
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-channel-syslog</artifactId>
<name>ROCKETMQ STREAMS :: channel-syslog</name>
diff --git a/rocketmq-streams-clients/pom.xml b/rocketmq-streams-clients/pom.xml
index e24e55f3..c21f7e01 100644
--- a/rocketmq-streams-clients/pom.xml
+++ b/rocketmq-streams-clients/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>rocketmq-streams</artifactId>
<groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/rocketmq-streams-commons/pom.xml b/rocketmq-streams-commons/pom.xml
index 2e73c6f8..d0985ab8 100755
--- a/rocketmq-streams-commons/pom.xml
+++ b/rocketmq-streams-commons/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-commons</artifactId>
diff --git a/rocketmq-streams-configurable/pom.xml b/rocketmq-streams-configurable/pom.xml
index ab90f715..fda51bdc 100755
--- a/rocketmq-streams-configurable/pom.xml
+++ b/rocketmq-streams-configurable/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-configurable</artifactId>
<name>ROCKETMQ STREAMS :: configurable</name>
diff --git a/rocketmq-streams-db-operator/pom.xml b/rocketmq-streams-db-operator/pom.xml
index fc5cb329..0b03e9db 100755
--- a/rocketmq-streams-db-operator/pom.xml
+++ b/rocketmq-streams-db-operator/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-db-operator</artifactId>
<name>ROCKETMQ STREAMS :: db-operator</name>
diff --git a/rocketmq-streams-examples/pom.xml b/rocketmq-streams-examples/pom.xml
index 7ced0cb0..e1f4b391 100644
--- a/rocketmq-streams-examples/pom.xml
+++ b/rocketmq-streams-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>rocketmq-streams</artifactId>
<groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/rocketmq-streams-filter/pom.xml b/rocketmq-streams-filter/pom.xml
index ca00220b..f57dacc5 100755
--- a/rocketmq-streams-filter/pom.xml
+++ b/rocketmq-streams-filter/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-filter</artifactId>
<name>ROCKETMQ STREAMS :: filter</name>
diff --git a/rocketmq-streams-script/pom.xml b/rocketmq-streams-script/pom.xml
index 7cd2b300..0d684515 100755
--- a/rocketmq-streams-script/pom.xml
+++ b/rocketmq-streams-script/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-script</artifactId>
<name>ROCKETMQ STREAMS :: script</name>
diff --git a/rocketmq-streams-serviceloader/pom.xml b/rocketmq-streams-serviceloader/pom.xml
index a1cf354d..d7742bbb 100755
--- a/rocketmq-streams-serviceloader/pom.xml
+++ b/rocketmq-streams-serviceloader/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-serviceloader</artifactId>
<packaging>jar</packaging>
diff --git a/rocketmq-streams-state/pom.xml b/rocketmq-streams-state/pom.xml
index 6e548a5f..8bf8088a 100644
--- a/rocketmq-streams-state/pom.xml
+++ b/rocketmq-streams-state/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-state</artifactId>
<packaging>jar</packaging>
diff --git a/rocketmq-streams-transport-minio/pom.xml b/rocketmq-streams-transport-minio/pom.xml
index 3637d333..73b706c9 100755
--- a/rocketmq-streams-transport-minio/pom.xml
+++ b/rocketmq-streams-transport-minio/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-transport-minio</artifactId>
<packaging>jar</packaging>
diff --git a/rocketmq-streams-window/pom.xml b/rocketmq-streams-window/pom.xml
index 8624079d..37d6093e 100755
--- a/rocketmq-streams-window/pom.xml
+++ b/rocketmq-streams-window/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
+ <version>1.0.3-preview-SNAPSHOT</version>
</parent>
<artifactId>rocketmq-streams-window</artifactId>
<packaging>jar</packaging>
[rocketmq-streams] 15/16: make RocketMQWindowExample runnable
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit aadf7eaa042ba3cc6c89433bc74bc99752c1c149
Author: 维章 <un...@gmail.com>
AuthorDate: Tue May 31 10:51:56 2022 +0800
make RocketMQWindowExample runnable
---
.../apache/rocketmq/streams/common/utils/KryoUtil.java | 4 ----
.../rocketmq/streams/common/utils/SerializeUtil.java | 6 ------
.../streams/window/operator/AbstractShuffleWindow.java | 17 ++---------------
.../rocketmq/streams/window/shuffle/ShuffleChannel.java | 9 +++++++++
4 files changed, 11 insertions(+), 25 deletions(-)
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/KryoUtil.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/KryoUtil.java
index 538cb027..d6fa1072 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/KryoUtil.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/KryoUtil.java
@@ -50,10 +50,6 @@ public class KryoUtil {
//不强制要求注册类（注册行为无法保证多个 JVM 内同一个类的注册编号相同；而且业务系统中大量的 Class 也难以一一注册）
kryo.setRegistrationRequired(false); //默认值就是 false，添加此行的目的是为了提醒维护者，不要改变这个配置
- //Fix the NPE bug when deserializing Collections.
-// ((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy())
-// .setFallbackInstantiatorStrategy(new StdInstantiatorStrategy());
-
return kryo;
}
};
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/SerializeUtil.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/SerializeUtil.java
index 7def6f7e..60691cc6 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/SerializeUtil.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/SerializeUtil.java
@@ -33,7 +33,6 @@ import org.apache.rocketmq.streams.common.datatype.ArrayDataType;
import org.apache.rocketmq.streams.common.datatype.DataType;
import org.apache.rocketmq.streams.common.datatype.StringDataType;
import org.apache.rocketmq.streams.common.interfaces.ISerialize;
-import org.nustaq.serialization.FSTConfiguration;
public class SerializeUtil {
private static final Log LOG = LogFactory.getLog(SerializeUtil.class);
@@ -44,11 +43,6 @@ public class SerializeUtil {
* @return
*/
public static byte[] serialize(Object object) {
- if(ISerialize.class.isInstance(object)){
-// byte[] bytes = conf.asByteArray(object);
-// return bytes;
- return KryoUtil.writeObjectToByteArray(object);
- }
DataType dataType = DataTypeUtil.getDataTypeFromClass(object.getClass());
if (ArrayDataType.class.isInstance(dataType)) {
int length = Array.getLength(object);
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
index ea3b923a..070a40f0 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
@@ -76,21 +76,7 @@ public abstract class AbstractShuffleWindow extends AbstractWindow {
@Override
public AbstractContext<IMessage> doMessage(IMessage message, AbstractContext context) {
- if (hasCreated.get()==false||this.shuffleChannel==null) {
- synchronized (this){
- if(hasCreated.get()==false||this.shuffleChannel==null){
- this.windowFireSource = new WindowTrigger(this);
- this.windowFireSource.init();
- this.windowFireSource.start(getFireReceiver());
- this.shuffleChannel = new ShuffleChannel(this);
- this.shuffleChannel.init();
- windowCache.setBatchSize(5000);
- windowCache.setShuffleChannel(shuffleChannel);
- shuffleChannel.startChannel();
- hasCreated.set(true);
- }
- }
- }
+ shuffleChannel.startChannel();
return super.doMessage(message, context);
}
@@ -99,6 +85,7 @@ public abstract class AbstractShuffleWindow extends AbstractWindow {
Set<String> splitIds = new HashSet<>();
splitIds.add(windowInstance.getSplitId());
shuffleChannel.flush(splitIds);
+
return doFireWindowInstance(windowInstance);
}
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
index 107f1ce5..a5c150a1 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
@@ -60,6 +60,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.rocketmq.streams.window.model.WindowCache.ORIGIN_MESSAGE_TRACE_ID;
@@ -110,6 +111,14 @@ public class ShuffleChannel extends AbstractSystemChannel {
}
+ protected transient AtomicBoolean hasStart = new AtomicBoolean(false);
+
+ @Override
+ public void startChannel() {
+ if (hasStart.compareAndSet(false, true)) {
+ super.startChannel();
+ }
+ }
/**
* init shuffle channel
[rocketmq-streams] 01/16: ConfigurableComponent insert and query
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 93d7f7753a5c03a25c93388d4d3cb34e2eaa79a9
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 23 10:45:02 2022 +0800
ConfigurableComponent insert and query
---
.../client/transform/window/WindowInfo.java | 3 +-
.../rocketmq/streams/client/DBDriverTest.java | 27 ++-----
.../rocketmq/streams/client/ORMUtilTest.java | 84 --------------------
.../org/apache/rocketmq/streams/client/Person.java | 89 ++++++++++++++++++++++
.../rocketmq/streams/common/utils/ReflectUtil.java | 2 +
5 files changed, 97 insertions(+), 108 deletions(-)
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
index e43e0943..73b543b3 100644
--- a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
@@ -27,8 +27,7 @@ public class WindowInfo {
public static int HOPPING_WINDOW = 1;//滑动窗口
public static int TUMBLING_WINDOW = 2;//滚动窗口
public static int SESSION_WINDOW = 3;
- public static int OVER_WINDOW = 4;
- public static int SHUFFLE_OVER_WINDOW = 5;
+
protected int type;//window类型 hopping，Tumbling
protected Time windowSize;//窗口大小
protected Time windowSlide;//滑动大小
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
index 0894c873..8d6ada4c 100644
--- a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
@@ -17,42 +17,25 @@
package org.apache.rocketmq.streams.client;
-import org.apache.rocketmq.streams.common.component.ComponentCreator;
-import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
-import org.apache.rocketmq.streams.configurable.model.Configure;
-import org.apache.rocketmq.streams.db.driver.DriverBuilder;
import org.junit.Test;
import static junit.framework.TestCase.assertNotNull;
-/**
- * 数据库的存储，需要配置存储的连接参数，请先完成配置，后执行单元用例 如果未建表，可以通过Configure.createTableSQL() 获取建表语句，创建表后，测试
- */
+
public class DBDriverTest {
- private String URL = "";
- protected String USER_NAME = "";
- protected String PASSWORD = "";
- protected String TABLE_NAME = "rocketmq_streams_configure_source";
@Test
public void testDBConfigurableService() {
String namespace = "streams.db.configurable";
- //正式使用时，在配置文件配置
- ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE, "DB");
- ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
- ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
- ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
- ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_TABLE_NAME, TABLE_NAME);
-
- //如果表不存在，创建表
- String sql = (Configure.createTableSQL(TABLE_NAME));
- DriverBuilder.createDriver().execute(sql);
- ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance("2211");
configurableComponent.insert(createPerson(namespace));
configurableComponent.refreshConfigurable(namespace);
Person person = configurableComponent.queryConfigurable("person", "peronName");
+ System.out.println(person.getName());
+ System.out.println(person.getAge());
+
assertNotNull(person);
}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
index 9b649d5f..ab2961a0 100644
--- a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
@@ -86,87 +86,3 @@ public class ORMUtilTest {
}
-class Person extends BasedConfigurable {
- @ENVDependence
- private String name;
- private int age;
- private Boolean isMale;
- private List<String> addresses;
- private Map<String, Integer> childName2Age;
-
- public static Person createPerson(String namespace) {
- Person person = new Person();
- person.setNameSpace(namespace);
- person.setType("person");
- person.setConfigureName("Chris");
- person.setName("Chris");
- List<String> addresses = new ArrayList<>();
- addresses.add("huilongguan");
- addresses.add("shangdi");
- person.setAddresses(addresses);
- Map<String, Integer> childName2Age = new HashMap<>();
- childName2Age.put("yuanyahan", 8);
- childName2Age.put("yuanruxi", 4);
- person.setChildName2Age(childName2Age);
- person.setMale(true);
- person.setAge(18);
- return person;
- }
-
- @Override
- public String toString() {
- return "org.apache.rocketmq.streams.Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
- + ", childName2Age=" + childName2Age + '}';
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public int getAge() {
- return age;
- }
-
- public void setAge(int age) {
- this.age = age;
- }
-
- public Boolean getMale() {
- return isMale;
- }
-
- public void setMale(Boolean male) {
- isMale = male;
- }
-
- public List<String> getAddresses() {
- return addresses;
- }
-
- public void setAddresses(List<String> addresses) {
- this.addresses = addresses;
- }
-
- public Map<String, Integer> getChildName2Age() {
- return childName2Age;
- }
-
- public void setChildName2Age(Map<String, Integer> childName2Age) {
- this.childName2Age = childName2Age;
- }
-
- @Override
- public Object clone() {
- Person person = null;
- try {
- person = (Person)super.clone();
- } catch (CloneNotSupportedException e) {
- System.out.println("clone error " + e);
- }
- return person;
- }
-}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/Person.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/Person.java
new file mode 100644
index 00000000..780d7d9d
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/Person.java
@@ -0,0 +1,89 @@
+package org.apache.rocketmq.streams.client;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+
+import java.util.List;
+import java.util.Map;
+
+public class Person extends BasedConfigurable {
+ @ENVDependence
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ @Override
+ public String toString() {
+ return "org.apache.rocketmq.streams.Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+
+ @Override
+ public Object clone() {
+ Person person = null;
+ try {
+ person = (Person) super.clone();
+ } catch (CloneNotSupportedException e) {
+ System.out.println("clone error " + e);
+ }
+ return person;
+ }
+}
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/ReflectUtil.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/ReflectUtil.java
index 77f1612c..de5a356f 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/ReflectUtil.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/ReflectUtil.java
@@ -679,6 +679,7 @@ public class ReflectUtil {
DataType dataType = DataTypeUtil.getDataTypeFromClass(fieldValue.getClass());
Object convertFieldValue = dataType.convert(fieldValue);
if (method != null) {
+ method.setAccessible(true);
method.invoke(object, convertFieldValue);
} else {
Field field = object.getClass().getDeclaredField(modelFieldName);
@@ -753,6 +754,7 @@ public class ReflectUtil {
if (method == null) {
throw new RuntimeException("can not get " + fieldName + "'s value, the method is not exist");
}
+ method.setAccessible(true);
return (T) method.invoke(bean);
} catch (Exception e) {
throw new RuntimeException("can not get " + fieldName + "'s value", e);
[rocketmq-streams] 07/16: Merge branch 'replaceDB' into devForEnterprise
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit ef6f7cf696075f66851d46b495e4acada4b679bd
Merge: 840c956f 4be93e83
Author: 维章 <un...@gmail.com>
AuthorDate: Thu May 26 15:28:53 2022 +0800
Merge branch 'replaceDB' into devForEnterprise
pom.xml | 2 +-
.../apache/rocketmq/streams/client/JoinTest.java | 17 ----
.../apache/rocketmq/streams/client/LeaseTest.java | 98 ----------------------
.../apache/rocketmq/streams/client/SplitTest.java | 86 -------------------
.../apache/rocketmq/streams/client/UnionTest.java | 23 -----
.../rocketmq/streams/client/example/DimTest.java | 54 ------------
.../streams/common/utils/CreateTopicUtil.java | 2 +-
.../rocketmq/streams/filter/operator/RuleTest.java | 39 ---------
.../window/storage/rocketmq/DefaultStorage.java | 5 +-
.../streams/window/WindowInstanceTest.java | 1 -
10 files changed, 3 insertions(+), 324 deletions(-)
[rocketmq-streams] 05/16: merge from upstream/snapshot-1.0.3
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit a539688121c9a3909af0818b9a01f5347548c256
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 23 16:09:07 2022 +0800
merge from upstream/snapshot-1.0.3
---
NOTICE | 2 +-
README.md | 16 +--
docs/README.md | 142 +++++++++++++++++++++
docs/SUMMARY.md | 8 ++
...225\264\344\275\223\346\236\266\346\236\204.md" | 33 +++++
.../2.\346\236\204\345\273\272DataStream.md" | 73 +++++++++++
.../3.\345\220\257\345\212\250DataStream.md" | 53 ++++++++
...265\201\350\275\254\350\277\207\347\250\213.md" | 63 +++++++++
...256\227\345\255\220\350\247\243\346\236\220.md" | 55 ++++++++
...256\236\347\216\260\345\256\271\351\224\231.md" | 0
"docs/images/Pipeline\347\261\273\345\233\276.png" | Bin 0 -> 44207 bytes
docs/images/img.png | Bin 38684 -> 0 bytes
docs/images/img_1.png | Bin 43711 -> 0 bytes
docs/images/img_2.png | Bin 103151 -> 0 bytes
docs/images/window.png | Bin 0 -> 241692 bytes
...75\223\346\236\266\346\236\204\345\233\276.png" | Bin 0 -> 60493 bytes
...00\273\344\275\223\350\277\207\347\250\213.png" | Bin 0 -> 44252 bytes
.../\346\211\251\345\256\271\345\211\215.png" | Bin 0 -> 56733 bytes
...12\266\346\200\201\347\256\227\345\255\220.png" | Bin 0 -> 35766 bytes
"docs/images/\347\212\266\346\200\201.png" | Bin 0 -> 47527 bytes
"docs/images/\347\274\251\345\256\271.png" | Bin 0 -> 51087 bytes
quick_start.md => docs/quick_start/README.md | 0
quick_start.md | 92 +++++++++----
.../window/operator/impl/SessionOperator.java | 8 ++
.../streams/window/operator/join/JoinWindow.java | 9 ++
stream_sink.md | 10 +-
stream_source.md | 2 +-
27 files changed, 524 insertions(+), 42 deletions(-)
diff --git a/NOTICE b/NOTICE
index 086ee9fa..a347efb1 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache RocketMQ
-Copyright 2016-2021 The Apache Software Foundation
+Copyright 2016-2022 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
index 51d9a6cf..f2f475e7 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,11 @@
-# Summary
-
+# RocketMQ Streams
+[](https://app.travis-ci.com/apache/rocketmq-streams)
+[](https://app.codecov.io/gh/apache/rocketmq-streams)
+[](https://github.com/apache/rocketmq-streams/releases)
+[](https://www.apache.org/licenses/LICENSE-2.0.html)
+[](http://isitmaintained.com/project/apache/rocketmq-streams "Average time to resolve an issue")
+[](http://isitmaintained.com/project/apache/rocketmq-streams "Percentage of issues still open")
+[](https://twitter.com/intent/follow?screen_name=ApacheRocketMQ)
## [中文文档](./README-chinese.md)
@@ -115,9 +121,3 @@ source
.start();
```
-=======
-* [Quick Start](quick\_start.md)
-* [创建实时任务数据源](stream\_source.md)
-* [创建实时任务数据输出](stream\_sink.md)
-* [数据处理逻辑](stream\_transform.md)
->>>>>>> 1cd2dd0291dbcab033e6773021ddca13ce819f82
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 00000000..9d5e464e
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,142 @@
+[](https://github.com/apache/rocketmq-streams/releases)
+[](https://www.apache.org/licenses/LICENSE-2.0.html)
+[](http://isitmaintained.com/project/apache/rocketmq-streams "Average time to resolve an issue")
+[](http://isitmaintained.com/project/apache/rocketmq-streams "Percentage of issues still open")
+[](https://twitter.com/intent/follow?screen_name=ApacheRocketMQ)
+
+# Features
+
+* 轻量级部署：可以单独部署，也支持集群部署
+* 多种类型的数据输入以及输出，source 支持 rocketmq ， sink 支持db, rocketmq 等
+
+# DataStream Example
+
+```java
+import org.apache.rocketmq.streams.client.transform.DataStream;
+
+DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+
+ source
+ .fromFile("～/admin/data/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
+```
+
+# Maven Repository
+
+```xml
+
+<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+</dependency>
+```
+
+# Core API
+
+rocketmq-stream 实现了一系列高级的API，可以让用户很方便的编写流计算的程序，实现自己的业务需求；
+
+## StreamBuilder
+
+StreamBuilder 用于构建流任务的源；
+
++ [dataStream(nameSpaceName,pipelineName)]() 返回DataStreamSource实例，用于分段编程实现流计算任务；
+
+## DataStream API
+
+### Source
+
+DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据；
+
++ ```fromFile``` 从文件中读取数据， 该方法包含俩个参数
+ + ```filePath``` 文件路径，必填参数
+ + ```isJsonData``` 是否json数据， 非必填参数， 默认为```true```
+
+
++ ```fromRocketmq``` 从rocketmq中获取数据，包含四个参数
+ + ```topic``` rocketmq消息队列的topic名称，必填参数
+ + ```groupName``` 消费者组的名称，必填参数
+ + ```isJson``` 是否json格式，非必填参数
+ + ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
+
++ ```fromMqtt``` 从满足MQTT协议的终端读取数据， 满足边缘计算的场景，其中包含9个参数
+ + ```url``` mqtt broker的地址，必填参数
+ + ```clientId``` 客户端ID， 必填参数，相同的clientId有负载的作用
+ + ```topic``` topic信息， 必填参数
+ + ```username``` 用户名, 非必填，在mqtt端添加鉴权机制时使用
+ + ```password``` 密码，非必填参数，在mqtt端添加鉴权机制时使用
+ + ```cleanSession``` 是否清理session信息， 非必填，默认为true
+ + ```connectionTimeout``` 连接超时信息， 非必填，默认是10s
+ + ```aliveInterval``` 判断连接是否活跃的间隔时间，非必填，默认是60s
+ + ```automaticReconnect``` 连接断开后自动重连机制，非必填，默认是true
+
+
++ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
+
+### transform
+
+transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
+
+DataStream实现了一系列常见的流计算算子
+
++ ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream
++ ```flatmap``` 与map类似，一个输入项对应0个或者多个输出项
++ ```filter``` 只选择func返回true的源DStream的记录来返回一个新的DStream
++ ```forEach``` 对每个记录执行一次函数func， 返回一个新的DataStream
++ ```selectFields``` 对每个记录返回对应的字段值，返回一个新的DataStream
++ ```operate``` 对每个记录执行一次自定义的函数，返回一个新的DataStream
++ ```script``` 针对每个记录的字段执行一段脚本，返回新的字段，生成一个新的DataStream
++ ```toPrint``` 将结果在控制台打印，生成新的DataStream实例
++ ```toFile``` 将结果保存为文件，生成一个新的DataStream实例
++ ```toMqtt``` 将结果输出到满足mqtt协议的设备中，生成一个新的DataStream实例
++ ```toDB``` 将结果保存到数据库
++ ```toRocketmq``` 将结果输出到rocketmq
++ ```to``` 将结果经过自定义的ISink接口输出到指定的存储
++ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
+ + ```count``` 在窗口内计数
+ + ```min``` 获取窗口内统计值的最小值
+ + ```max``` 获取窗口内统计值得最大值
+ + ```avg``` 获取窗口内统计值的平均值
+ + ```sum``` 获取窗口内统计值的加和值
+ + ```reduce``` 在窗口内进行自定义的汇总运算
++ ```join``` 根据条件将俩个流进行内关联
++ ```leftJoin``` 根据条件将俩个流的数据进行左关联
++ ```dimJoin``` 根据条件将流与维表进行内关联，维表的数据可以来自于文件，也可以来自于数据库
++ ```dimLeftJoin``` 根据条件将流与维表进行左关联，维表的数据可以来自于文件，也可以来自于数据库
++ ```union``` 将俩个流进行合并
++ ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算
++ ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等
+
+#### Strategy
+
+策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
+
+```java
+//指定checkpoint的存储策略
+source
+ .fromRocketmq("TSG_META_INFO","")
+ .map(message->message+"--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
+ .start();
+```
+
+# 运行
+
+Rocketmq-Streams 作为典型的java应用，既可以集成在业务系统里运行，也可以作为一个独立的jar包来运行；
+
+首先对应用的源码进行编译
+
+```shell
+mvn -Prelease-all -DskipTests clean install -U
+```
+
+然后直接通过java指令来运行
+
+```shell
+ java -jar jarName mainClass
+```
+
+更多详细的案例可以看[这里](docs/SUMMARY.md)
\ No newline at end of file
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
new file mode 100644
index 00000000..950f99bd
--- /dev/null
+++ b/docs/SUMMARY.md
@@ -0,0 +1,8 @@
+# Summary
+
+* [Introduction](README.md)
+* [Quick Start](quick_start/README.md)
+* [创建实时任务数据源](stream_source/README.md)
+* [创建实时任务数据输出](stream_sink/README.md)
+* [数据处理逻辑](stream_transform/README.md)
+
diff --git "a/docs/design/1.RocketMQ-streams\346\225\264\344\275\223\346\236\266\346\236\204.md" "b/docs/design/1.RocketMQ-streams\346\225\264\344\275\223\346\236\266\346\236\204.md"
new file mode 100644
index 00000000..8e564a8c
--- /dev/null
+++ "b/docs/design/1.RocketMQ-streams\346\225\264\344\275\223\346\236\266\346\236\204.md"
@@ -0,0 +1,33 @@
+### 总体架构
+
+
+
+数据从RocketMQ中被RocketMQ-streams消费，经过处理最终被写回到RocketMQ。
+如果流处理任务中含有算子groupBy，则需要将数据按照Key进行分组，将分组数据写入shuffle topic。后续算子从
+shuffle topic消费。如果还涉及count之类有状态算子，那么计算时需要读写状态，在窗口触发之后将计算结果写出。
+
+
+### 任务并行度模型
+
+
+
+计算实例实质上是依赖了Rocket-streams SDK的client，因此，计算实例消费的MQ依赖RocketMQ rebalance分配，
+计算实例总个数也不能大于消费总MQ个数，否则将有部分计算实例处于等待状态，消费不到数据。
+
+一个计算实例可以消费多个MQ，一个实例内也只有一张计算拓扑图。
+
+### 状态
+
+
+对于有状态算子，他的状态本地依赖RocksDB加速读取，远程依赖Mysql做持久化。允许流计算任务时，可以只依赖本地存储
+RocksDB, 只需要将setLocalStorageOnly设置成true即可。这种情况下可能存在状态丢失。
+
+
+
+### 扩缩容
+
+
+
+当计算实例从3个缩容到2个，借助于RocketMQ的rebalance，MQ会在计算实例之间重新分配。
+Instance1上消费的MQ2和MQ3被分配到Instance2和Instance3上，这两个MQ的状态数据也需要迁移到Instance2
+和Instance3上，这也暗示，状态数据是根据源数据分片保存的；扩容则是刚好相反的过程。
diff --git "a/docs/design/2.\346\236\204\345\273\272DataStream.md" "b/docs/design/2.\346\236\204\345\273\272DataStream.md"
new file mode 100644
index 00000000..af12e27a
--- /dev/null
+++ "b/docs/design/2.\346\236\204\345\273\272DataStream.md"
@@ -0,0 +1,73 @@
+DataStreamSource中有一个PipelineBuilder，在后续构建过程中，这个PipelineBuilder会一直向后流传，
+将构建过程中产生的source、stage添加进来；最后在start的时候，真正利用PipelineBuilder构建出拓扑图。
+
+### source类型
+ - 设置source的namespace、configureName;
+ - 将source保存到PipelineBuilder中;
+ - 将source作为source节点保存到PipelineBuilder中的ChainPipeline中;
+
+### ChainStage类型
+
+所有的其他运算，包括map,filter,script，window都会先构建出ChainStage，然后以ChainStage的身份进入
+PipelineBuilder，参加后续构建。
+
+在DataStream中一个典型的添加新算子，过程如下所示：
+```java
+
+public DataStream script(String script) {
+ //将用户定义的cript转化成ChainStage
+ // ChainStage<?> stage = this.mainPipelineBuilder.createStage(new ScriptOperator(script));
+ //将ChainStage添加到PipelineBuilder中，构建拓扑。
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ //将PipelineBuilder构建成DataStream，向后传递，后续还可以用该PipelineBuilder构建拓扑
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+}
+
+```
+
+### 创建ChainStage
+
+PipelineBuilder创建，创建过程中会设置label，并将这个ChainStage添加到PipelineBuilder持有ChainPipeline中
+
+- 把ChainStage添加到pipeline中
+ 在构建过程中，所有的添加算子都使用一个共同的PipelineBuilder实例，PipelineBuilder结构如图所示，他持有
+ 一个ChainPipeline实例，ChainPipeline实例中含有一个ISource和多个stages，还有一个label与stage的映射关系，
+ 以及用于寻找下个stage的label。
+ 
+在createStage过程中，将chainStage加入到Pipeline中。
+
+在setTopologyStages 过程中将label加入到Pipeline中；
+
+### 设置拓扑
+```java
+public void setTopologyStages(ChainStage currentChainStage, List<ChainStage> nextStages) {
+ if (isBreak) {
+ return;
+ }
+ if (nextStages == null) {
+ return;
+ }
+ List<String> lableNames = new ArrayList<>();
+ for (ChainStage stage : nextStages) {
+ lableNames.add(stage.getLabel());
+ }
+
+ if (currentChainStage == null) {
+ this.pipeline.setChannelNextStageLabel(lableNames);
+ } else {
+ currentChainStage.setNextStageLabels(lableNames);
+ for (ChainStage stage : nextStages) {
+ stage.getPrevStageLabels().add(currentChainStage.getLabel());
+ }
+ }
+ }
+```
+
+如果是首个ChainStage，则设置下一跳的label；如果不是首个，需要将下个stage的label设置进入当前stage。
+同时，下个stage也需要设置前一个stage的label标签。形成双向链表的结构。
+
+
+
+
+
+
diff --git "a/docs/design/3.\345\220\257\345\212\250DataStream.md" "b/docs/design/3.\345\220\257\345\212\250DataStream.md"
new file mode 100644
index 00000000..91f91f2f
--- /dev/null
+++ "b/docs/design/3.\345\220\257\345\212\250DataStream.md"
@@ -0,0 +1,53 @@
+### Start流程
+
+流式计算在运行时可以拉起多个相同实例进行扩容，所以不能直接启动上述已经构建好的拓扑图，需要将上述构建好的拓扑
+图保存起来，需要扩容时，直接拿出算子的副本，实例化启动即可。
+
+### 统一管理点
+
+- 加载统一管理点IConfigurableService;
+
+ 三种方式存储：Memory, db, file
+
+- PipelineBuilder的build方法，将构建构成中保存起来的IConfigurable，source和statge都是IConfigurable，
+ 保存到IConfigurableService中;
+
+- IConfigurableService的refreshConfigurable方法;
+
+ 1.主要做的事可以概括：从统一管理点加载出组件，赋值，init，在调用后置方法doProcessAfterRefreshConfigurable。
+
+ 2.ChainPipeline的后置方法比较特殊，会调用pipeline中各个组件的后置方法，如果这个组件是普通UDFChainStage，
+ 那么将会反序列化，实例成StageBuilder。如果是WindowChainStage，会讲用户数据接收的window实例化出来。
+
+ 3.从IConfigurable中加载实例副本出来；
+
+ 4.将实例副本赋值；
+
+ 5.初始化实例副本，实例都是AbstractConfigurable的继承类，调用他的的init方法。比如在初始化rocketmqSource
+ 的时候，就会在此时调用init方法，先于启动方法调用；
+
+ 6.调用IConfigurable的doProcessAfterRefreshConfigurable方法，目前只有ChainPipeline会调用，
+ （典型的是ChainPipeline），会在此方法中构建label与stage映射的stageMap；设置source；再调用
+ ChainPipeline中各个stage的doProcessAfterRefreshConfigurable方法；
+
+ 7.这里ChainPipeline的stage都是UDFChainStage类似。UDFChainStage的
+ doProcessAfterRefreshConfigurable方法会将之前序列化好的StageBuilder反序列化，成为StageBuilder实例。
+
+ 8.如果这个stage是window类型的WindowChainStage，ChainPipeline调用各个stage的
+ doProcessAfterRefreshConfigurable。这里会将用于数据接收的window实例化赋值；
+
+ 9.OutputChainStage此时会从统一管理点IConfigurableService查询出sink实例，并赋值给自己sink字段；
+
+
+### ChainPipeline的启动
+```java
+pipeline.startChannel();
+```
+
+将ChainPipeline作为整个数据接收的入口，并启动source；
+
+当source有数据进来时，ChainPipeline将会收到数据；具体方法是ChainPipeline的doMessageInner方法；
+
+该方法将数据封装承AbstractContext后，向后传递；
+
+
diff --git "a/docs/design/4.\346\225\260\346\215\256\347\232\204\346\265\201\350\275\254\350\277\207\347\250\213.md" "b/docs/design/4.\346\225\260\346\215\256\347\232\204\346\265\201\350\275\254\350\277\207\347\250\213.md"
new file mode 100644
index 00000000..1eb791cb
--- /dev/null
+++ "b/docs/design/4.\346\225\260\346\215\256\347\232\204\346\265\201\350\275\254\350\277\207\347\250\213.md"
@@ -0,0 +1,63 @@
+###总体过程
+
+
+
+数据流转整体过程如图所示，黑色箭头线是数据流，橙色为控制流。数据的整体流向是从source中接收到，经过
+AbstractSource判断是否发出系统消息，在进入ChainPipeline,ChainPipeline根据之间构建好的处理拓扑图，使用
+深度优先策略找出下一个处理节点stage，交给Pipeline。Pipeline发现如果是系统消息则对stage执行特殊的控制逻辑，
+如果不是，则用stage来处理具体数据。
+
+### 无window算子执行流程
+- source从RocketMQ中消费数据，进入RocketMQSource的父类AbstractSource；
+- AbstractSource启动控制流，判断是否数据来自新分片，如果是，首先向下游传递一条NewSplitMessage消息，等待系
+ 统消息处理完成返回后，才能继续处理该数据。
+- NewSplitMessage进入Pipeline，如果是系统消息，stage执行该类系统消息对应的控制操作。如果不是系统消息则用
+stage处理数据；
+- Pipeline执行完成后，返回到ChainPipeline，选择下一个stage继续执行；
+- 遍历stage直到结束。
+
+### 含有window算子执行流程
+
+
+
+- 数据流和控制流在上述流程一致，即先进入source，然后由AbstractSource判断是否发出发出系统消息，再进入
+ ChainPipeline按照已经构建好的拓扑图执行。
+- 不同的是，如果是window算子，那么这条数据在执行具体计算之前需要先按照groupBy分组，在执行算子，例如count。
+分组操作需要借助于shuffle topic完成，即写入shuffle topic之前先按照groupBy的值，计算数据写入目的
+ MessageQueue，相同groupBy值的数据将被写入一个MessageQueue中。这样shuffle数据被读取时，
+ groupBy值相同的数据总会被一个client处理，达到按照groupBy分组处理的效果。
+
+- ShuffleChannel会自动订阅、消费shuffle topic。数据会经过shuffle并在ShuffleChannel中再次被消费到。
+- 判断是否是系统消息，如果是，执行该种类系统消息对应的控制流操作。
+- 如果不是系统消息，触发window中算子计算，比如算子是count，就对某个key出现的次数加1；count算子用到的状
+ 态会在接收到NewSplitMessage类型系统消息时提前加载好。计算结束后的状态保存到RocksDB或者mysql中。
+
+- window到时间后，将计算结果输出到下游stage继续计算，并清理RocksDB、Mysql中对应的状态。
+
+
+### 系统消息
+
+#### NewSplitMessage
+当发现数据来自新分片（MessageQueue）时，由AbstractSource产生并向下游拓扑传递。
+
+作用于window算子，使其提前加载该分片对应的状态数据到内存，使得状态数据对该分片数据进行计算时，能使用
+到对应的状态，得出正确的结果。
+
+#### CheckPointMessage
+
+##### 产生时机：
+- 消费分片移除时；
+- RocketMQ-streams向broker提交消费offset时；
+- 处理完一批次消息后；
+
+##### 作用
+- 作用于各个缓存，例如将数据写入shuffle topic之前的WindowCache，使缓存中数据写出到下游。
+- 作用于sink，将sink中缓存而未写出的数据写出；
+- 将有状态算子的状态flush到存储；
+
+#### RemoveSplitMessage
+比较RocketMQ client触发rebalance前后消费的分片，如果某个分片不在被消费，需要将该分片移除，在移除该分配时发出
+RemoveSplitMessage类型消息。
+
+作用于window算子，将RocksDB中状态清除；
+
diff --git "a/docs/design/5.Window\347\256\227\345\255\220\350\247\243\346\236\220.md" "b/docs/design/5.Window\347\256\227\345\255\220\350\247\243\346\236\220.md"
new file mode 100644
index 00000000..4e8be748
--- /dev/null
+++ "b/docs/design/5.Window\347\256\227\345\255\220\350\247\243\346\236\220.md"
@@ -0,0 +1,55 @@
+### window算子初始化
+window的实例化和初始化时机，与普通无状态算子一样，在构建DataStream阶段以stage形式加入pipeline。在启动
+DataStream阶段完成window的初始化。
+
+
+
+- 给window初始化WindowStorage用户状态存储；
+
+ WindowStorage包括localStorage存储和remoteStorage存储；localStorage使用RocksDB，
+ remoteStorage使用mysql；
+
+- 向window添加一个WindowCache的匿名实例，用于存储写入shuffle topic之前数据；
+- 向window添加SQLCache，作为写入Mysql之前的缓存；
+- 向window添加ShuffleChannel，作为写出shuffle和接收来自shufffle topic数据的通道；
+
+
+### ShuffleChannel写出shuffle数据
+AbstractShuffleWindow的doMessage方法，将数据写入shuffleChannel
+```java
+public AbstractContext<IMessage> doMessage(IMessage message, AbstractContext context) {
+ shuffleChannel.startChannel();
+ return super.doMessage(message, context);
+}
+```
+
+- shuffleChannel.startChannel
+启动shuffleChannel中的consumer，从shuffletopic中消费数据；如果有消费到数据，将由
+ shuffleChannel的doMessage处理。
+
+- AbstractWindow.doMessage方法
+
+对于一条消息来说，window 首先需要检查是否有窗口实例，如果没有则创建。如果窗口实例已经超过最大的watermark，
+数据丢弃，否则进行消息积累 消息会先经历batchAdd 然后flush加入到windowCache中；windowCache定时触发，加入到
+shuffleMsgCache中，shuffleMsgCache中定时发出，用shuffleMsgCache中的producer写出到rocketmq。
+
+### ShuffleChannel接收到shuffle数据
+ShuffleChannel#doMessage方法；
+
+将shuffle消息加入到shuffleCache中
+
+最终进入ShuffleCache#batchInsert中
+
+WindowOperator#shuffleCalculate中
+
+实际窗口计算：WindowValue#calculate
+
+计算后并不会马上触发窗口，窗口需要定时出发
+
+### window触发
+ WindowFireSource#startSource启动定时任务，1s检查一次窗口是否触发WindowFireSource#fireWindowInstance
+WindowOperator#fireWindowInstance
+
+windowFireSource.executeMessage
+
+windowFireSource.executeMessage这个方法里面会执行pipeline的下个节点
\ No newline at end of file
diff --git "a/docs/design/6.RocketMQ-streams\345\246\202\344\275\225\345\256\236\347\216\260\345\256\271\351\224\231.md" "b/docs/design/6.RocketMQ-streams\345\246\202\344\275\225\345\256\236\347\216\260\345\256\271\351\224\231.md"
new file mode 100644
index 00000000..e69de29b
diff --git "a/docs/images/Pipeline\347\261\273\345\233\276.png" "b/docs/images/Pipeline\347\261\273\345\233\276.png"
new file mode 100644
index 00000000..dafe81a1
Binary files /dev/null and "b/docs/images/Pipeline\347\261\273\345\233\276.png" differ
diff --git a/docs/images/img.png b/docs/images/img.png
deleted file mode 100644
index b814adf7..00000000
Binary files a/docs/images/img.png and /dev/null differ
diff --git a/docs/images/img_1.png b/docs/images/img_1.png
deleted file mode 100644
index 16a45cc1..00000000
Binary files a/docs/images/img_1.png and /dev/null differ
diff --git a/docs/images/img_2.png b/docs/images/img_2.png
deleted file mode 100644
index 0b75ab05..00000000
Binary files a/docs/images/img_2.png and /dev/null differ
diff --git a/docs/images/window.png b/docs/images/window.png
new file mode 100644
index 00000000..30ba8945
Binary files /dev/null and b/docs/images/window.png differ
diff --git "a/docs/images/\346\200\273\344\275\223\346\236\266\346\236\204\345\233\276.png" "b/docs/images/\346\200\273\344\275\223\346\236\266\346\236\204\345\233\276.png"
new file mode 100644
index 00000000..5eba9cec
Binary files /dev/null and "b/docs/images/\346\200\273\344\275\223\346\236\266\346\236\204\345\233\276.png" differ
diff --git "a/docs/images/\346\200\273\344\275\223\350\277\207\347\250\213.png" "b/docs/images/\346\200\273\344\275\223\350\277\207\347\250\213.png"
new file mode 100644
index 00000000..7a68947e
Binary files /dev/null and "b/docs/images/\346\200\273\344\275\223\350\277\207\347\250\213.png" differ
diff --git "a/docs/images/\346\211\251\345\256\271\345\211\215.png" "b/docs/images/\346\211\251\345\256\271\345\211\215.png"
new file mode 100644
index 00000000..5232b762
Binary files /dev/null and "b/docs/images/\346\211\251\345\256\271\345\211\215.png" differ
diff --git "a/docs/images/\346\234\211\347\212\266\346\200\201\347\256\227\345\255\220.png" "b/docs/images/\346\234\211\347\212\266\346\200\201\347\256\227\345\255\220.png"
new file mode 100644
index 00000000..a9ec479c
Binary files /dev/null and "b/docs/images/\346\234\211\347\212\266\346\200\201\347\256\227\345\255\220.png" differ
diff --git "a/docs/images/\347\212\266\346\200\201.png" "b/docs/images/\347\212\266\346\200\201.png"
new file mode 100644
index 00000000..e2fd9b2e
Binary files /dev/null and "b/docs/images/\347\212\266\346\200\201.png" differ
diff --git "a/docs/images/\347\274\251\345\256\271.png" "b/docs/images/\347\274\251\345\256\271.png"
new file mode 100644
index 00000000..05dcee41
Binary files /dev/null and "b/docs/images/\347\274\251\345\256\271.png" differ
diff --git a/quick_start.md b/docs/quick_start/README.md
similarity index 100%
copy from quick_start.md
copy to docs/quick_start/README.md
diff --git a/quick_start.md b/quick_start.md
index a60dbb95..adcb529d 100644
--- a/quick_start.md
+++ b/quick_start.md
@@ -1,46 +1,84 @@
-# 快速开发
+## rocketmq-streams 快速搭建
+---
-## 引入相关的jar包
+### 前言
-```xml
+本文档主要介绍如何基于rocketmq-streams快速搭建流处理任务，搭建过程中某些例子会用到rocketmq，可以参考[rocketmq搭建文档](https://rocketmq.apache.org/docs/quick-start/)
-<dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-clients</artifactId>
-</dependency>
+### 1、源码构建
-```
+#### 1.1、构建环境
-## 开发实时应用程序
+- JDK 1.8 and above
+- Maven 3.2 and above
-```java
+#### 1.2、构建Rocketmq-streams
-public class RocketmqExample {
+```shell
+git clone https://github.com/apache/rocketmq-streams.git
+cd rocketmq-streams
+mvn clean -DskipTests install -U
- public static void main(String[] args) {
+```
- DataStreamSource dataStream = StreamBuilder.dataStream("test_namespace", "graph_pipeline");
+### 2、基于rocketmq-streams创建应用
- dataStream
- .fromFile("data.csv", false) //构建实时任务的数据源
- .map(message -> message.split(",")) //构建实时任务处理的逻辑过程
- .toPrint(1) //构建实时任务的输出
- .start(); //启动实时任务
- }
-}
+#### 2.1、pom依赖
+```xml
+
+<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-clients</artifactId>
+</dependency>
```
-## 运行
+#### 2.2、shade clients依赖包
-打包
+```xml
-```shell
-mvn -Prelease-all -DskipTests clean install -U
+<build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <minimizeJar>false</minimizeJar>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <artifactSet>
+ <includes>
+ <include>org.apache.rocketmq:rocketmq-streams-clients</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+</build>
```
-运行
+#### 2.3、编写业务代码
+
+快速编写一个统计页面点击次数的小程序：Please see the [rocketmq-streams-examples](rocketmq-streams-examples/README.md)
+
+#### 2.4、运行
+
+- 前提：在从rocketmq中读取数据做流处理时，需要运行topic在rocketmq中自动创建，因为做groupBy操作时，需要用到rocketmq作为shuffle数据的读写目的地。
+- 命令：
-```shell
- java -jar jarName mainClass
```
+ java -jar XXXX-shade.jar \
+ -Dlog4j.level=ERROR \
+ -Dlog4j.home=/logs \
+ -Xms1024m \
+ -Xmx1024m
+```
+
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java
index 0c10a9bf..14e1ffd3 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/impl/SessionOperator.java
@@ -524,4 +524,12 @@ public class SessionOperator extends WindowOperator {
}
return numer;
}
+
+ public int getSessionTimeOut() {
+ return sessionTimeOut;
+ }
+
+ public void setSessionTimeOut(int sessionTimeOut) {
+ this.sessionTimeOut = sessionTimeOut;
+ }
}
\ No newline at end of file
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java
index f356a286..103f40e3 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/join/JoinWindow.java
@@ -60,6 +60,7 @@ public class JoinWindow extends AbstractShuffleWindow {
protected String joinType;//join类型，值为INNER,LEFT
protected String expression;//条件表达式。在存在非等值比较时使用
+ protected String rightDependentTableName;
@Override
protected int doFireWindowInstance(WindowInstance instance) {
@@ -557,4 +558,12 @@ public class JoinWindow extends AbstractShuffleWindow {
public void setExpression(String expression) {
this.expression = expression;
}
+
+ public String getRightDependentTableName() {
+ return rightDependentTableName;
+ }
+
+ public void setRightDependentTableName(String rightDependentTableName) {
+ this.rightDependentTableName = rightDependentTableName;
+ }
}
diff --git a/stream_sink.md b/stream_sink.md
index d7718865..a30aae59 100644
--- a/stream_sink.md
+++ b/stream_sink.md
@@ -54,7 +54,7 @@
String topic=.....; //rocketmq 的topic
String namesrvAddress=......; //rocketmq的nameserver
- DataStream dataStream=dataStream.toRocketmq(topic,namesrvAddress);
+ DataStream dataStream=dataStreamSource.toRocketmq(topic,namesrvAddress);
```
@@ -65,7 +65,7 @@
String topic=.....; //rocketmq 的topic
String groupName=.....; // rocketmq的消费组
String namesrvAddress=......; //rocketmq的nameserver
- DataStream dataStream=dataStream.toRocketmq(topic,groupName,namesrvAddress);
+ DataStream dataStream=dataStreamSource.toRocketmq(topic,groupName,namesrvAddress);
```
@@ -77,7 +77,7 @@
String groupName=.....; // rocketmq的消费组
String namesrvAddress=......; //rocketmq的nameserver
String tags=......; // rocketmq的tag信息
- DataStream dataStream=dataStream.toRocketmq(topic,tags,groupName,namesrvAddress);
+ DataStream dataStream=dataStreamSource.toRocketmq(topic,tags,groupName,namesrvAddress);
```
## kafka
@@ -95,7 +95,7 @@
String url=......;
String clientId=......;
String topic=......;
- DataStream dataStream=dataStream.toMqtt(url,cliientId,topic);
+ DataStream dataStream=dataStreamSource.toMqtt(url,cliientId,topic);
```
@@ -108,7 +108,7 @@
String topic=......;
String username=......;
String password=......;
- DataStream dataStream=dataStream.toMqtt(url,cliientId,topic,username,password);
+ DataStream dataStream=dataStreamSource.toMqtt(url,cliientId,topic,username,password);
```
diff --git a/stream_source.md b/stream_source.md
index 8ae7b7e4..77fb96c4 100644
--- a/stream_source.md
+++ b/stream_source.md
@@ -54,7 +54,7 @@
String groupName = .....; // rocketmq的消费组
String namesrvAddress = ......; //rocketmq的nameserver
Boolean isJsonData = true; //是否json
- String tags = ......; //rocketmq的tag信息
+ String tags = ......; // rocketmq的tag信息
DataStream dataStream = dataStreamSource.fromRocketmq(topic, groupName, tags, isJsonData, namesrvAddress);
```
[rocketmq-streams] 11/16: make FileSourceExample runnable
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 4d94b411e56b98cc2fada8013a686ebe0bc59962
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 30 15:03:58 2022 +0800
make FileSourceExample runnable
---
.../apache/rocketmq/streams/examples/source/FileSourceExample.java | 2 +-
rocketmq-streams-examples/src/main/resources/joinData-2.txt | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/source/FileSourceExample.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/source/FileSourceExample.java
index ecaf0bca..b51ea2eb 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/source/FileSourceExample.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/source/FileSourceExample.java
@@ -22,7 +22,7 @@ import org.apache.rocketmq.streams.client.source.DataStreamSource;
public class FileSourceExample {
public static void main(String[] args) {
DataStreamSource source = StreamBuilder.dataStream("namespace", "pipeline");
- source.fromFile("/Users/junjie.cheng/jobs/access.log", false)
+ source.fromFile("data.txt", false)
.map(message -> message)
.toPrint(1)
.start();
diff --git a/rocketmq-streams-examples/src/main/resources/joinData-2.txt b/rocketmq-streams-examples/src/main/resources/joinData-2.txt
new file mode 100644
index 00000000..a711532a
--- /dev/null
+++ b/rocketmq-streams-examples/src/main/resources/joinData-2.txt
@@ -0,0 +1,4 @@
+{"InFlow":"10","ProjectName":"ProjectName-0","LogStore":"LogStore-0","OutFlow":"10"}
+{"InFlow":"20","ProjectName":"ProjectName-10","LogStore":"LogStore-10","OutFlow":"10"}
+
+
[rocketmq-streams] 13/16: modify topic
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 0bcaf5bf92b03448b22eb472da9a01c0b52df6f3
Merge: 4d94b411 5a236975
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 30 15:11:56 2022 +0800
modify topic
.../apache/rocketmq/streams/examples/join/RocketmqJoinExample.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --cc rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
index 31b3b831,30dec798..8d79e076
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
@@@ -22,10 -22,10 +22,11 @@@ import org.apache.rocketmq.streams.clie
public class RocketmqJoinExample {
public static void main(String[] args) {
DataStream left = StreamBuilder.dataStream("tmp", "tmp")
- .fromRocketmq("TopicTest", "groupA", true, "localhost:9876");
+ .fromRocketmq("TopicTestJoin", "groupA", true, "localhost:9876");
DataStream right = StreamBuilder.dataStream("tmp", "tmp22")
- .fromRocketmq("TopicTest", "groupB", true, "localhost:9876");
+ .fromRocketmq("TopicTestJoin", "groupB", true, "localhost:9876");
+
+
left.join(right)
.on("(ProjectName,=,ProjectName)")
.toDataSteam()
[rocketmq-streams] 09/16: fix(example) join example
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit ff92d26b808f1f4695535cc5d98ba56e37d74237
Author: 维章 <un...@gmail.com>
AuthorDate: Fri May 27 18:52:25 2022 +0800
fix(example) join example
---
.../org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
index f4270d8a..31b3b831 100644
--- a/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
+++ b/rocketmq-streams-examples/src/main/java/org/apache/rocketmq/streams/examples/join/RocketmqJoinExample.java
@@ -23,7 +23,7 @@ public class RocketmqJoinExample {
public static void main(String[] args) {
DataStream left = StreamBuilder.dataStream("tmp", "tmp")
.fromRocketmq("TopicTest", "groupA", true, "localhost:9876");
- DataStream right = StreamBuilder.dataStream("tmp", "tmp")
+ DataStream right = StreamBuilder.dataStream("tmp", "tmp22")
.fromRocketmq("TopicTest", "groupB", true, "localhost:9876");
left.join(right)
[rocketmq-streams] 14/16: remove mini batch
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 24b5bf902a1f1faa15b1eabd0df6c44d0d9f9d07
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 30 15:39:02 2022 +0800
remove mini batch
---
.../apache/rocketmq/streams/sink/RocketMQSink.java | 30 ++--
.../window/minibatch/MiniBatchMsgCache.java | 58 -------
.../rocketmq/streams/window/model/WindowCache.java | 182 +++++++++++++--------
.../window/operator/AbstractShuffleWindow.java | 1 -
.../streams/window/shuffle/ShuffleChannel.java | 2 -
5 files changed, 128 insertions(+), 145 deletions(-)
diff --git a/rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java b/rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java
index 1de92e31..d702053e 100644
--- a/rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java
+++ b/rocketmq-streams-channel-rocketmq/src/main/java/org/apache/rocketmq/streams/sink/RocketMQSink.java
@@ -241,30 +241,22 @@ public class RocketMQSink extends AbstractSupportShuffleSink {
}
@Override
- public List<ISplit<?, ?>> getSplitList() {
+ public List<ISplit<?,?>> getSplitList() {
initProducer();
- List<ISplit<?, ?>> messageQueues = new ArrayList<>();
- List<MessageQueue> metaqQueueSet = new ArrayList<>();
+ List<ISplit<?,?>> messageQueues = new ArrayList<>();
try {
- if (messageQueues == null || messageQueues.size() == 0) {
- try {
- metaqQueueSet = producer.fetchPublishMessageQueues(topic);
- }catch (Exception e) {
- producer.send(new Message(topic, "test", "test".getBytes(StandardCharsets.UTF_8)));
- metaqQueueSet = producer.fetchPublishMessageQueues(topic);
- }
- List<ISplit<?, ?>> queueList = new ArrayList<>();
- for (MessageQueue queue : metaqQueueSet) {
- RocketMQMessageQueue rocketMQMessageQueue = new RocketMQMessageQueue(queue);
- queueList.add(rocketMQMessageQueue);
+ List<MessageQueue> messageQueueSet = producer.fetchPublishMessageQueues(topic);
+ List<ISplit<?,?>> queueList = new ArrayList<>();
+ for (MessageQueue queue : messageQueueSet) {
+ RocketMQMessageQueue rocketMQMessageQueue = new RocketMQMessageQueue(queue);
+ queueList.add(rocketMQMessageQueue);
- }
- queueList.sort((Comparator<ISplit>) Comparable::compareTo);
- messageQueues = queueList;
}
- } catch (Exception e) {
- throw new RuntimeException(e);
+ queueList.sort((Comparator<ISplit>) Comparable::compareTo);
+ messageQueues = queueList;
+ } catch (MQClientException e) {
+ return messageQueues;
}
return messageQueues;
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/minibatch/MiniBatchMsgCache.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/minibatch/MiniBatchMsgCache.java
deleted file mode 100644
index a36309e2..00000000
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/minibatch/MiniBatchMsgCache.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.streams.window.minibatch;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.rocketmq.streams.common.channel.sinkcache.IMessageFlushCallBack;
-import org.apache.rocketmq.streams.common.channel.sinkcache.impl.AbstractMultiSplitMessageCache;
-import org.apache.rocketmq.streams.common.channel.sinkcache.impl.MessageCache;
-import org.apache.rocketmq.streams.common.channel.split.ISplit;
-import org.apache.rocketmq.streams.common.context.IMessage;
-import org.apache.rocketmq.streams.common.topology.shuffle.IShuffleKeyGenerator;
-import org.apache.rocketmq.streams.window.operator.AbstractShuffleWindow;
-
-public class MiniBatchMsgCache extends AbstractMultiSplitMessageCache<Pair<ISplit,IMessage>> {
- public static String SHUFFLE_KEY="shuffle_key";
-
-
-
- protected transient IShuffleKeyGenerator shuffleKeyGenerator;
- protected transient AbstractShuffleWindow window;
-
-
-
-
- public MiniBatchMsgCache(
- IMessageFlushCallBack<Pair<ISplit,IMessage>> flushCallBack, IShuffleKeyGenerator shuffleKeyGenerator,
- AbstractShuffleWindow window) {
- super(flushCallBack);
- this.shuffleKeyGenerator=shuffleKeyGenerator;
- this.window=window;
- }
-
-
- @Override protected String createSplitId(Pair<ISplit, IMessage> msg) {
- return msg.getLeft().getQueueId();
- }
-
- @Override protected MessageCache createMessageCache() {
- ShuffleMessageCache messageCache=new ShuffleMessageCache(this.flushCallBack);
- messageCache.setWindow(window);
- messageCache.setShuffleKeyGenerator(shuffleKeyGenerator);
- return messageCache;
- }
-}
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowCache.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowCache.java
index c53a50ff..8ce1232f 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowCache.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/model/WindowCache.java
@@ -19,9 +19,14 @@ package org.apache.rocketmq.streams.window.model;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
@@ -29,30 +34,26 @@ import org.apache.commons.logging.LogFactory;
import org.apache.rocketmq.streams.common.batchsystem.BatchFinishMessage;
import org.apache.rocketmq.streams.common.channel.sink.AbstractSink;
import org.apache.rocketmq.streams.common.channel.sinkcache.IMessageFlushCallBack;
+import org.apache.rocketmq.streams.common.channel.sinkcache.impl.AbstractMultiSplitMessageCache;
import org.apache.rocketmq.streams.common.channel.split.ISplit;
import org.apache.rocketmq.streams.common.component.ComponentCreator;
import org.apache.rocketmq.streams.common.context.IMessage;
import org.apache.rocketmq.streams.common.context.Message;
import org.apache.rocketmq.streams.common.topology.model.IWindow;
-import org.apache.rocketmq.streams.common.topology.shuffle.IShuffleKeyGenerator;
import org.apache.rocketmq.streams.common.utils.CompressUtil;
-import org.apache.rocketmq.streams.window.minibatch.MiniBatchMsgCache;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.window.debug.DebugWriter;
import org.apache.rocketmq.streams.window.shuffle.ShuffleChannel;
-import org.apache.rocketmq.streams.window.util.ShuffleUtil;
/**
* 缓存数据，flush时，刷新完成数据落盘
*/
-public abstract class WindowCache extends
- AbstractSink implements IWindow.IWindowCheckpoint {
-
+public abstract class WindowCache extends AbstractSink implements IWindow.IWindowCheckpoint {
private static final Log LOG = LogFactory.getLog(WindowCache.class);
- public static final String SPLIT_SIGN = "##";
public static final String IS_COMPRESSION_MSG = "_is_compress_msg";
public static final String COMPRESSION_MSG_DATA = "_compress_msg";
- public static final String MSG_FROM_SOURCE = "msg_from_source";
public static final String ORIGIN_OFFSET = "origin_offset";
public static final String ORIGIN_QUEUE_ID = "origin_queue_id";
@@ -66,94 +67,105 @@ public abstract class WindowCache extends
public static final String SHUFFLE_KEY = "SHUFFLE_KEY";
public static final String ORIGIN_MESSAGE_TRACE_ID = "origin_request_id";
+
protected transient boolean isWindowTest = false;
+
protected transient AtomicLong COUNT = new AtomicLong(0);
/**
* 分片转发channel
*/
protected transient ShuffleChannel shuffleChannel;
- public void initMiniBatch() {
- shuffleMsgCache = new MiniBatchMsgCache(new WindowCache.MutilMsgMergerAndCompressFlushCallBack(),(IShuffleKeyGenerator) shuffleChannel.getWindow(),shuffleChannel.getWindow());
-
-
- shuffleMsgCache.openAutoFlush();
- }
-
-// protected transient AtomicLong insertCount=new AtomicLong(0);
-// protected transient AtomicLong shuffleCount=new AtomicLong(0);
-// protected transient AtomicLong SUM=new AtomicLong(0);
-
- protected class MutilMsgMergerAndCompressFlushCallBack implements IMessageFlushCallBack<Pair<ISplit, IMessage>> {
+ protected class ShuffleMsgCache extends AbstractMultiSplitMessageCache<Pair<ISplit, JSONObject>> {
+
+ public ShuffleMsgCache() {
+ super(new IMessageFlushCallBack<Pair<ISplit, JSONObject>>() {
+ @Override
+ public boolean flushMessage(List<Pair<ISplit, JSONObject>> messages) {
+ if (messages == null || messages.size() == 0) {
+ return true;
+ }
+ ISplit split = messages.get(0).getLeft();
+ JSONObject jsonObject = messages.get(0).getRight();
+ JSONArray allMsgs = shuffleChannel.getMsgs(jsonObject);
+ for (int i = 1; i < messages.size(); i++) {
+ Pair<ISplit, JSONObject> pair = messages.get(i);
+ JSONObject msg = pair.getRight();
+ JSONArray jsonArray = shuffleChannel.getMsgs(msg);
+ if (jsonArray != null) {
+ allMsgs.addAll(jsonArray);
+ }
+ }
+ JSONObject zipJsonObject = new JSONObject();
+ zipJsonObject.put(COMPRESSION_MSG_DATA, CompressUtil.gZip(jsonObject.toJSONString()));
+ zipJsonObject.put(IS_COMPRESSION_MSG, true);
+ shuffleChannel.getProducer().batchAdd(new Message(zipJsonObject), split);
+ shuffleChannel.getProducer().flush(split.getQueueId());
+ return true;
+ }
+ });
+ }
@Override
- public boolean flushMessage(List<Pair<ISplit, IMessage>> messages) {
- if (messages == null || messages.size() == 0) {
- return true;
- }
- long start=System.currentTimeMillis();
- ISplit split = messages.get(0).getLeft();
- JSONArray allMsgs =new JSONArray();
- long sum=0;
- for (int i = 0; i < messages.size(); i++) {
- Pair<ISplit, IMessage> pair = messages.get(i);
- IMessage message = pair.getRight();
- allMsgs.add(message.getMessageBody());
- // sum=SUM.addAndGet(message.getMessageBody().getLong("total"));
- }
- //System.out.println("before shuffle sum is "+sum);
- JSONObject jsonObject=shuffleChannel.createMsg(allMsgs,split);
-// JSONObject zipJsonObject = new JSONObject();
-// zipJsonObject.put(COMPRESSION_MSG_DATA, CompressUtil.gZip(jsonObject.toJSONString()));
-// zipJsonObject.put(IS_COMPRESSION_MSG, true);
- shuffleChannel.getProducer().batchAdd(new Message(jsonObject), split);
- shuffleChannel.getProducer().flush(split.getQueueId());
- long cost=System.currentTimeMillis()-start;
- // shuffleCount.addAndGet(cost);
- return true;
+ protected String createSplitId(Pair<ISplit, JSONObject> msg) {
+ return msg.getLeft().getQueueId();
}
}
-
- protected transient MiniBatchMsgCache shuffleMsgCache ;
+ protected transient ShuffleMsgCache shuffleMsgCache = new ShuffleMsgCache();
@Override
protected boolean initConfigurable() {
-
+ shuffleMsgCache = new ShuffleMsgCache();
+ shuffleMsgCache.setBatchSize(1000);
+ shuffleMsgCache.setAutoFlushSize(100);
+ shuffleMsgCache.setAutoFlushTimeGap(1000);
+ shuffleMsgCache.openAutoFlush();
isWindowTest = ComponentCreator.getPropertyBooleanValue("window.fire.isTest");
return super.initConfigurable();
}
@Override
protected boolean batchInsert(List<IMessage> messageList) {
- long start=System.currentTimeMillis();
- for (IMessage msg : messageList) {
- String shuffleKey = generateShuffleKey(msg);
- IMessage message= ShuffleUtil.createShuffleMsg(msg,shuffleKey);
- if(message==null){
- continue;
+ Map<Integer, JSONArray> shuffleMap = translateToShuffleMap(messageList);
+ if (shuffleMap != null && shuffleMap.size() > 0) {
+ Set<String> splitIds = new HashSet<>();
+
+ for (Map.Entry<Integer, JSONArray> entry : shuffleMap.entrySet()) {
+ ISplit split = shuffleChannel.getSplit(entry.getKey());
+ JSONObject msg = shuffleChannel.createMsg(entry.getValue(), split);
+
+ shuffleMsgCache.addCache(new MutablePair<>(split, msg));
+ List<IMessage> messages = new ArrayList<>();
+ splitIds.add(split.getQueueId());
+
+ if (DebugWriter.getDebugWriter(shuffleChannel.getWindow().getConfigureName()).isOpenDebug()) {
+ JSONArray jsonArray = entry.getValue();
+ for (int i = 0; i < jsonArray.size(); i++) {
+ Message message = new Message(jsonArray.getJSONObject(i));
+ message.getHeader().setQueueId(jsonArray.getJSONObject(i).getString(ORIGIN_QUEUE_ID));
+ message.getHeader().setOffset(jsonArray.getJSONObject(i).getLong(ORIGIN_OFFSET));
+ messages.add(message);
+
+ }
+ DebugWriter.getDebugWriter(shuffleChannel.getWindow().getConfigureName()).writeWindowCache(shuffleChannel.getWindow(), messages, split.getQueueId());
+ }
+
}
- addPropertyToMessage(msg, message.getMessageBody());
- Integer index = shuffleChannel.hash(shuffleKey);
- ISplit split = shuffleChannel.getSplit(index);
- shuffleMsgCache.addCache(new MutablePair(split, message));
+
}
if (isWindowTest) {
long count = COUNT.addAndGet(messageList.size());
System.out.println(shuffleChannel.getWindow().getConfigureName() + " send shuffle msg count is " + count);
shuffleMsgCache.flush();
}
- long cost=System.currentTimeMillis()-start;
- //shuffleCount.addAndGet(cost);
return true;
}
@Override
public void finishBatchMsg(BatchFinishMessage batchFinishMessage) {
- long start=System.currentTimeMillis();
if (shuffleChannel != null && shuffleChannel.getProducer() != null) {
- this.flush();
- shuffleMsgCache.flush();
+ shuffleChannel.getProducer().flush();
for (ISplit split : shuffleChannel.getQueueList()) {
IMessage message = batchFinishMessage.getMsg().deepCopy();
message.getMessageBody().put(ORIGIN_QUEUE_ID, message.getHeader().getQueueId());
@@ -161,11 +173,51 @@ public abstract class WindowCache extends
}
shuffleChannel.getProducer().flush();
}
- // System.out.println("insert cost is "+insertCount.get()+" shuffle cost is "+shuffleCount.get()+" finish batch cost is "+(System.currentTimeMillis()-start));
}
+ /**
+ * 对接收的消息按照不同shuffle key进行分组
+ *
+ * @param messages
+ * @return
+ */
+ protected Map<Integer, JSONArray> translateToShuffleMap(List<IMessage> messages) {
+ Map<Integer, JSONArray> shuffleMap = new HashMap<>();
+ for (IMessage msg : messages) {
+ if (msg.getHeader().isSystemMessage()) {
+ continue;
+ }
+ String shuffleKey = generateShuffleKey(msg);
+ if (StringUtil.isEmpty(shuffleKey)) {
+ shuffleKey = "<null>";
+ LOG.debug("there is no group by value in message! " + msg.getMessageBody().toString());
+ //continue;
+ }
+ Integer index = shuffleChannel.hash(shuffleKey);
+ JSONObject body = msg.getMessageBody();
+ String offset = msg.getHeader().getOffset();
+ String queueId = msg.getHeader().getQueueId();
+
+ body.put(ORIGIN_OFFSET, offset);
+ body.put(ORIGIN_QUEUE_ID, queueId);
+ body.put(ORIGIN_QUEUE_IS_LONG, msg.getHeader().getMessageOffset().isLongOfMainOffset());
+ body.put(ORIGIN_MESSAGE_HEADER, JSONObject.toJSONString(msg.getHeader()));
+ body.put(ORIGIN_MESSAGE_TRACE_ID, msg.getHeader().getTraceId());
+ body.put(SHUFFLE_KEY, shuffleKey);
+
+ addPropertyToMessage(msg, body);
+
+ JSONArray jsonArray = shuffleMap.get(index);
+ if (jsonArray == null) {
+ jsonArray = new JSONArray();
+ shuffleMap.put(index, jsonArray);
+ }
+ jsonArray.add(body);
+ }
+ return shuffleMap;
+ }
/**
* 根据message生成shuffle key
@@ -195,8 +247,8 @@ public abstract class WindowCache extends
return shuffleChannel;
}
- public MiniBatchMsgCache getShuffleMsgCache() {
- return this.shuffleMsgCache;
+ public ShuffleMsgCache getShuffleMsgCache() {
+ return shuffleMsgCache;
}
public void setShuffleChannel(ShuffleChannel shuffleChannel) {
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
index 7c78f478..ea3b923a 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/operator/AbstractShuffleWindow.java
@@ -86,7 +86,6 @@ public abstract class AbstractShuffleWindow extends AbstractWindow {
this.shuffleChannel.init();
windowCache.setBatchSize(5000);
windowCache.setShuffleChannel(shuffleChannel);
- windowCache.initMiniBatch();
shuffleChannel.startChannel();
hasCreated.set(true);
}
diff --git a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
index d834af41..107f1ce5 100644
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/shuffle/ShuffleChannel.java
@@ -47,7 +47,6 @@ import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
import org.apache.rocketmq.streams.common.utils.StringUtil;
import org.apache.rocketmq.streams.common.utils.TraceUtil;
import org.apache.rocketmq.streams.window.debug.DebugWriter;
-import org.apache.rocketmq.streams.window.minibatch.MiniBatchMsgCache;
import org.apache.rocketmq.streams.window.model.WindowCache;
import org.apache.rocketmq.streams.window.model.WindowInstance;
import org.apache.rocketmq.streams.window.operator.AbstractShuffleWindow;
@@ -61,7 +60,6 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.rocketmq.streams.window.model.WindowCache.ORIGIN_MESSAGE_TRACE_ID;
[rocketmq-streams] 02/16: merge 0.1
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit f5a1380ffeb6fa18920f59edf54b39d996caf77b
Merge: d4243be9 1cd2dd02
Author: 维章 <un...@gmail.com>
AuthorDate: Mon May 23 11:53:01 2022 +0800
merge 0.1
NOTICE | 2 +-
README.md | 16 +-
SUMMARY.md | 7 +
docs/README.md | 142 ------
docs/SUMMARY.md | 8 -
...225\264\344\275\223\346\236\266\346\236\204.md" | 33 --
.../2.\346\236\204\345\273\272DataStream.md" | 73 ----
.../3.\345\220\257\345\212\250DataStream.md" | 53 ---
...265\201\350\275\254\350\277\207\347\250\213.md" | 63 ---
...256\227\345\255\220\350\247\243\346\236\220.md" | 55 ---
...256\236\347\216\260\345\256\271\351\224\231.md" | 0
"docs/images/Pipeline\347\261\273\345\233\276.png" | Bin 44207 -> 0 bytes
docs/images/img.png | Bin 0 -> 38684 bytes
docs/images/img_1.png | Bin 0 -> 43711 bytes
docs/images/img_2.png | Bin 0 -> 103151 bytes
docs/images/window.png | Bin 241692 -> 0 bytes
...75\223\346\236\266\346\236\204\345\233\276.png" | Bin 60493 -> 0 bytes
...00\273\344\275\223\350\277\207\347\250\213.png" | Bin 44252 -> 0 bytes
.../\346\211\251\345\256\271\345\211\215.png" | Bin 56733 -> 0 bytes
...12\266\346\200\201\347\256\227\345\255\220.png" | Bin 35766 -> 0 bytes
"docs/images/\347\212\266\346\200\201.png" | Bin 47527 -> 0 bytes
"docs/images/\347\274\251\345\256\271.png" | Bin 51087 -> 0 bytes
docs/quick_start/README.md | 46 --
pom.xml | 124 ++----
quick_start.md | 92 ++--
.../pom.xml | 43 +-
rocketmq-streams-cep/src/test/resources/log4j.xml | 36 ++
rocketmq-streams-channel-db/pom.xml | 6 +-
.../streams/db/sink/AbstractMultiTableSink.java | 12 +-
.../streams/db/sink/DynamicMultipleDBSink.java | 11 +-
.../streams/db/sink/SelfMultiTableSink.java | 2 +-
.../streams/db/sink/SplitBySerialNumber.java | 2 +-
.../streams/db/sink/SplitByTimeMultiTableSink.java | 2 +-
rocketmq-streams-channel-es/pom.xml | 29 +-
.../{ESSinkBuilder.java => ESChannelBuilder.java} | 54 +--
.../rocketmq/streams/es/sink/ESSinkBuilder.java | 1 +
.../streams/es/sink/ESSinkOnlyChannel.java | 43 +-
.../apache/rocketmq/streams/es/sink/EsClient.java | 135 ++++++
rocketmq-streams-channel-http/pom.xml | 6 +-
rocketmq-streams-channel-kafka/pom.xml | 32 ++
.../streams/kafka/KafkaChannelBuilder.java | 52 ++-
.../apache/rocketmq/streams/kafka/KafkaSplit.java | 55 +++
.../rocketmq/streams/kafka/sink/KafkaSink.java | 200 +++++++++
.../rocketmq/streams/kafka/source/KafkaSource.java | 238 ++++++++++
.../rocketmq/streams/kafka/KafkaChannelTest.java | 104 +++++
.../src/test/resources/log4j.xml | 20 +
rocketmq-streams-channel-mqtt/pom.xml | 24 +-
.../rocketmq/streams/mqtt/source/PahoSource.java | 41 +-
rocketmq-streams-channel-rocketmq/pom.xml | 30 +-
.../apache/rocketmq/streams/debug/DebugWriter.java | 92 ++--
.../apache/rocketmq/streams/sink/RocketMQSink.java | 42 +-
.../rocketmq/streams/source/RocketMQSource.java | 14 +-
.../rocketmq/streams/RocketMQChannelTest.java | 2 +-
rocketmq-streams-channel-syslog/pom.xml | 21 +-
.../rocketmq/streams/syslog/SyslogChannel.java | 35 +-
.../streams/syslog/SyslogChannelBuilder.java | 4 +
.../streams/syslog/SyslogChannelManager.java | 6 +-
.../rocketmq/streams/syslog/SyslogServer.java | 35 +-
.../rocketmq/streams/syslog/SyslogClient.java | 22 +-
rocketmq-streams-checkpoint/pom.xml | 64 ---
.../streams/checkpoint/db/DBCheckPointStorage.java | 68 ---
rocketmq-streams-clients/pom.xml | 23 +-
.../streams/client/source/DataStreamSource.java | 20 +
.../streams/client/transform/DataStream.java | 43 +-
.../streams/client/transform/JoinStream.java | 5 +-
.../streams/client/transform/SplitStream.java | 24 +-
.../streams/client/transform/WindowStream.java | 50 ++-
.../rocketmq/streams/client/ApplicationTest.java | 57 +++
.../rocketmq/streams/client/MqttSourceExample.java | 80 ++++
.../{sink/UserDefinedSink.java => ScriptTest.java} | 27 +-
.../apache/rocketmq/streams/client/WindowTest.java | 14 +-
.../rocketmq/streams/client/example/JoinTest.java | 7 +-
.../rocketmq/streams/client/example/SplitTest.java | 31 +-
.../streams/client/sink/UserDefinedSink.java | 2 +-
.../client/sink/UserDefinedSupportShuffleSink.java | 4 +-
rocketmq-streams-commons/pom.xml | 48 +-
.../MappedByteBufferTableWithPrimaryIndex.java | 482 +++++++++++++++++++++
.../streams/common/cache/compress/KVAddress.java | 48 +-
.../streams/common/channel/AbstractChannel.java | 14 +-
.../AbstractSupportShuffleChannelBuilder.java | 2 +-
.../common/channel/builder/IChannelBuilder.java | 25 +-
.../channel/builder/IShuffleChannelBuilder.java | 4 +-
.../common/channel/impl/CollectionSink.java | 39 +-
.../common/channel/impl/CollectionSinkBuilder.java | 34 +-
.../common/channel/impl/PrintChannelBuilder.java | 30 +-
.../channel/impl/file/FileChannelBuilder.java | 45 +-
.../streams/common/channel/impl/file/FileSink.java | 4 +-
.../common/channel/impl/file/FileSource.java | 2 +-
.../channel/impl/memory/MemoryChannelBuilder.java | 64 +++
.../common/channel/impl/memory/MemorySink.java | 4 +-
.../channel/impl/view/ViewChannelBuilder.java | 43 +-
.../streams/common/channel/impl/view/ViewSink.java | 31 +-
.../common/channel/impl/view/ViewSource.java | 66 +++
.../streams/common/channel/sink/AbstractSink.java | 17 +-
.../channel/sink/AbstractSupportShuffleSink.java | 8 +-
.../sink/AbstractSupportShuffleUDFSink.java | 6 +-
.../common/channel/sink/AbstractUDFSink.java | 23 +-
.../streams/common/channel/sink/ISink.java | 9 +-
.../impl/AbstractMultiSplitMessageCache.java | 22 +-
.../common/channel/source/AbstractSource.java | 94 ++--
.../channel/source/AbstractUnreliableSource.java | 2 +-
.../streams/common/channel/split/ISplit.java | 5 +-
.../streams/common/component/ComponentCreator.java | 4 +
.../common/configurable/AbstractConfigurable.java | 17 +-
.../common/configurable/BasedConfigurable.java | 13 +-
.../common/configurable/IConfigurableService.java | 5 +-
.../streams/common/configure/ConfigureFileKey.java | 7 +-
.../streams/common/context/AbstractContext.java | 33 +-
.../streams/common/context/MessageHeader.java | 89 ++--
.../streams/common/datatype/IntDataType.java | 3 +-
.../streams/common/datatype/ShortDataType.java | 2 +-
.../streams/common/interfaces/ISerialize.java | 8 +-
.../rocketmq/streams/common/model/NameCreator.java | 14 +-
.../streams/common/model/NameCreatorContext.java | 31 +-
.../common/monitor/ConsoleMonitorManager.java | 412 ++++++++++++++++++
.../streams/common/monitor/DataSyncConstants.java | 54 +++
.../streams/common/monitor/HttpClient.java | 116 +++++
.../rocketmq/streams/common/monitor/HttpUtil.java | 248 +++++++++++
.../monitor/MonitorDataSyncServiceFactory.java | 61 +++
.../common/monitor/group/MonitorCommander.java | 4 +-
.../streams/common/monitor/impl/DipperMonitor.java | 2 +-
.../streams/common/monitor/model/JobStage.java | 350 +++++++++++++++
.../streams/common/monitor/model/TraceIdsDO.java | 126 ++++++
.../common/monitor/model/TraceMonitorDO.java | 250 +++++++++++
.../monitor/service/MonitorDataSyncService.java | 23 +-
.../service/impl/DBMonitorDataSyncImpl.java | 63 +++
.../service/impl/HttpMonitorDataSyncImpl.java | 151 +++++++
.../service/impl/RocketMQMonitorDataSyncImpl.java | 185 ++++++++
.../optimization/IHomologousOptimization.java | 2 +-
.../common/optimization/MessageGlobleTrace.java | 16 +-
.../streams/common/optimization/Re2Engine.java | 47 +-
.../common/optimization/TaskOptimization.java | 17 +-
.../optimization/fingerprint/FingerprintCache.java | 4 +-
.../optimization/fingerprint/PreFingerprint.java | 54 ++-
.../streams/common/schedule/ScheduleTask.java | 4 +-
.../common/threadpool/ThreadPoolFactory.java | 34 +-
.../AbstractMutilPipelineChainPipline.java | 81 ++--
.../streams/common/topology/ChainPipeline.java | 255 ++++++-----
.../streams/common/topology/ChainStage.java | 32 +-
.../common/topology/builder/PipelineBuilder.java | 139 ++++--
.../common/topology/metric/NotFireReason.java | 176 ++++++++
.../streams/common/topology/metric/StageGroup.java | 248 +++++++++++
.../common/topology/metric/StageMetric.java | 138 ++++++
.../common/topology/model/AbstractStage.java | 177 +++++---
.../streams/common/topology/model/IWindow.java | 3 +-
.../streams/common/topology/model/Pipeline.java | 56 ++-
.../topology/model/PipelineSourceJoiner.java | 48 --
.../topology/shuffle/IShuffleKeyGenerator.java | 16 +-
.../common/topology/shuffle/ShuffleMQCreator.java | 401 ++++++++++-------
.../topology/stages/AbstractWindowStage.java | 5 +-
.../stages/EmptyChainStage.java} | 27 +-
.../common/topology/stages/FilterChainStage.java | 157 ++-----
.../common/topology/stages/JoinChainStage.java | 3 +-
.../common/topology/stages/JoinEndChainStage.java | 11 +-
.../topology/stages/JoinStartChainStage.java | 67 +++
.../common/topology/stages/OutputChainStage.java | 81 ++--
.../topology/stages/ShuffleConsumerChainStage.java | 193 +++++++++
.../topology/stages/ShuffleProducerChainStage.java | 345 +++++++++++++++
.../topology/stages/SubPiplineChainStage.java | 138 ------
.../common/topology/stages/UnionChainStage.java | 3 +-
.../common/topology/stages/UnionEndChainStage.java | 10 +-
...onChainStage.java => UnionStartChainStage.java} | 43 +-
.../ViewChainStage.java} | 479 ++++++++++----------
.../common/topology/stages/udf/UDFChainStage.java | 27 +-
.../streams/common/topology/task/StreamsTask.java | 444 +++----------------
.../streams/common/topology/task/TaskAssigner.java | 20 +-
.../streams/common/utils/ConfigurableUtil.java | 6 +-
.../streams/common/utils/ContantsUtil.java | 33 +-
.../streams/common/utils/DataTypeUtil.java | 16 +-
.../rocketmq/streams/common/utils/FileUtil.java | 89 +++-
.../streams/common/utils/InstantiationUtil.java | 48 +-
.../streams/common/utils/JsonableUtil.java | 5 +
.../rocketmq/streams/common/utils/KryoUtil.java | 214 +++++++++
.../streams/common/utils/NameCreatorUtil.java | 60 ---
.../streams/common/utils/PipelineHTMLUtil.java | 299 +++++++++++++
.../streams/common/utils/PropertiesUtils.java | 2 +-
.../rocketmq/streams/common/utils/ReflectUtil.java | 77 ++--
.../streams/common/utils/SerializeUtil.java | 23 +-
.../streams/common/utils/ServiceLoadUtil.java | 63 +++
.../rocketmq/streams/common/utils/TraceUtil.java | 23 +-
.../rocketmq/streams/common/channel/SinkTest.java | 4 +-
rocketmq-streams-configurable/pom.xml | 11 +-
.../configurable/ConfigurableComponent.java | 17 +-
.../service/AbstractConfigurableService.java | 98 ++---
.../service/impl}/FileConfigureService.java | 2 +-
.../impl}/FileSupportParentConfigureService.java | 2 +-
.../service/impl/HttpConfigureService.java | 377 ++++++++++++++++
.../impl/HttpSupportParentConfigureService.java | 20 +-
.../service/impl}/MemoryConfigureService.java | 2 +-
.../impl}/MemorySupportParentConfigureService.java | 2 +-
rocketmq-streams-connectors/pom.xml | 40 --
.../connectors/balance/AbstractBalance.java | 207 ---------
.../streams/connectors/balance/ISourceBalance.java | 60 ---
.../streams/connectors/balance/SplitChanged.java | 55 ---
.../connectors/balance/impl/LeaseBalanceImpl.java | 144 ------
.../streams/connectors/model/PullMessage.java | 50 ---
.../streams/connectors/model/ReaderStatus.java | 120 -----
.../streams/connectors/reader/DBScanReader.java | 269 ------------
.../streams/connectors/reader/ISplitReader.java | 96 ----
.../connectors/reader/SplitCloseFuture.java | 83 ----
.../connectors/source/AbstractPullSource.java | 272 ------------
.../source/CycleDynamicMultipleDBScanSource.java | 213 ---------
.../source/DynamicMultipleDBScanSource.java | 190 --------
.../streams/connectors/source/IPullSource.java | 60 ---
.../source/filter/BoundedPatternFilter.java | 53 ---
.../source/filter/CyclePatternFilter.java | 173 --------
.../connectors/source/filter/CyclePeriod.java | 222 ----------
.../connectors/source/filter/CycleSchedule.java | 236 ----------
.../source/filter/DataFormatPatternFilter.java | 106 -----
.../connectors/source/filter/PatternFilter.java | 41 --
rocketmq-streams-db-operator/pom.xml | 6 +-
.../streams/db/configuable/DBConfigureService.java | 11 +-
rocketmq-streams-dbinit/pom.xml | 6 +-
.../dbinit/mysql/delegate/DBDelegateFactory.java | 5 +
.../dbinit/mysql/delegate/MysqlDelegate.java | 1 +
.../src/main/resources/tables_mysql_innodb.sql | 16 -
rocketmq-streams-dim/pom.xml | 6 +-
.../intelligence/AbstractIntelligenceCache.java | 5 +-
rocketmq-streams-examples/pom.xml | 6 +-
.../checkpoint/RemoteCheckpointExample.java | 4 +-
.../mutilconsumer/MultiStreamsExample.java | 4 +-
.../streams/examples/source/FileSourceExample.java | 2 +-
.../examples/source/RocketmqSourceExample4.java | 63 ---
.../src/main/resources/joinData-1.txt | 4 -
.../src/main/resources/joinData-2.txt | 4 -
rocketmq-streams-filter/pom.xml | 5 +-
.../streams/filter/builder/ExpressionBuilder.java | 49 +--
.../streams/filter/context/RuleContext.java | 31 +-
.../filter/engine/impl/DefaultRuleEngine.java | 29 +-
.../function/expression/CompareFunction.java | 16 +-
.../rocketmq/streams/filter/operator/Rule.java | 58 +++
.../expression/ExpressionRelationParser.java | 9 +-
.../expression/ExpressionRelationPaser.java | 107 -----
.../operator/expression/GroupExpression.java | 3 +-
.../operator/expression/RelationExpression.java | 35 +-
.../PiplineLogFingerprintAnalysis.java | 6 +-
.../dependency/BlinkRuleV2Expression.java | 5 +-
.../optimization/dependency/DependencyTree.java | 22 +-
.../dependency/SimplePipelineTree.java | 3 +-
.../dependency/StateLessDependencyTree.java | 84 ++++
.../optimization/homologous/HomologousCompute.java | 5 +-
.../homologous/HomologousOptimization.java | 13 +-
rocketmq-streams-lease/pom.xml | 6 +-
rocketmq-streams-runner/assembly/distribution.xml | 69 ---
rocketmq-streams-runner/assembly/standalone.xml | 72 ---
rocketmq-streams-runner/bin/start.sh | 58 ---
rocketmq-streams-runner/bin/stop.sh | 33 --
rocketmq-streams-runner/pom.xml | 80 ----
.../src/main/resources/log4j.xml | 51 ---
rocketmq-streams-schedule/pom.xml | 6 +-
.../schedule/job/ConfigurableExecutorJob.java | 30 +-
rocketmq-streams-script/pom.xml | 6 +-
.../function/aggregation/LastValueAccumulator.java | 67 +++
.../function/impl/distinct/DistinctFunction.java | 1 -
.../function/impl/json/JsonCreatorFunction.java | 4 +-
.../function/impl/json/UDTFFieldNameFunction.java | 50 +++
.../script/function/impl/parser/GrokFunction.java | 4 +-
.../function/impl/parser/Paser2JsonFunction.java | 17 +-
.../function/impl/parser/PaserBySplitFunction.java | 44 +-
.../function/impl/parser/RegexParserFunction.java | 24 +-
.../script/function/impl/router/RouteFunction.java | 4 +-
.../script/operator/impl/AggregationScript.java | 19 +-
.../streams/script/service/IAccumulator.java | 4 +-
.../script/service/udf/SimpleUDAFScript.java | 29 +-
.../streams/script/service/udf/UDFScript.java | 26 +-
.../streams/script/function/FunctionTest.java | 19 +-
rocketmq-streams-serviceloader/pom.xml | 6 +-
rocketmq-streams-state/pom.xml | 6 +-
.../streams/state/kv/rocksdb/RocksDBOperator.java | 37 +-
rocketmq-streams-transport-minio/pom.xml | 6 +-
rocketmq-streams-window/pom.xml | 7 +-
.../window/minibatch/MiniBatchMsgCache.java | 76 ++++
.../window/minibatch/ShuffleMessageCache.java | 187 ++++++++
.../rocketmq/streams/window/model/WindowCache.java | 182 +++-----
.../streams/window/model/WindowInstance.java | 19 +-
.../window/offset/WindowMaxValueManager.java | 8 +-
.../window/operator/AbstractShuffleWindow.java | 36 +-
.../streams/window/operator/AbstractWindow.java | 74 +++-
.../window/operator/impl/SessionOperator.java | 3 +-
.../window/operator/impl/WindowOperator.java | 25 +-
.../streams/window/operator/join/JoinWindow.java | 39 +-
.../window/shuffle/AbstractSystemChannel.java | 97 +++--
.../streams/window/shuffle/ShuffleCache.java | 13 +-
.../streams/window/shuffle/ShuffleChannel.java | 74 ++--
.../streams/window/state/impl/WindowValue.java | 54 ++-
.../window/storage/AbstractWindowStorage.java | 10 +-
.../window/storage/ShufflePartitionManager.java | 8 +-
.../window/storage/rocksdb/RocksdbStorage.java | 7 +-
.../streams/window/trigger/WindowTrigger.java | 2 +-
.../rocketmq/streams/window/util/ShuffleUtil.java | 62 +++
.../org/apache/rocketmq/streams/RocksdbTest.java | 2 +-
docs/stream_sink/README.md => stream_sink.md | 19 +-
docs/stream_source/README.md => stream_source.md | 30 +-
.../README.md => stream_transform.md | 0
294 files changed, 9733 insertions(+), 7012 deletions(-)
diff --cc README.md
index f2f475e7,e4205699..51d9a6cf
--- a/README.md
+++ b/README.md
@@@ -1,123 -1,6 +1,123 @@@
- # RocketMQ Streams
- [](https://app.travis-ci.com/apache/rocketmq-streams)
- [](https://app.codecov.io/gh/apache/rocketmq-streams)
- [](https://github.com/apache/rocketmq-streams/releases)
- [](https://www.apache.org/licenses/LICENSE-2.0.html)
- [](http://isitmaintained.com/project/apache/rocketmq-streams "Average time to resolve an issue")
- [](http://isitmaintained.com/project/apache/rocketmq-streams "Percentage of issues still open")
- [](https://twitter.com/intent/follow?screen_name=ApacheRocketMQ)
+ # Summary
+
+
+## [中文文档](./README-chinese.md)
+
+## [Quick Start](./quick_start.md)
+
+## Features
+
+* Lightweight deployment: RocketMQ Streams can be deployed separately or in cluster mode.
+* Various types of data input and output: source supports [RocketMQ](https://github.com/apache/rocketmq) while sink supports databases and RocketMQ, etc.
+
+## DataStream Example
+
+```java
+import org.apache.rocketmq.streams.client.transform.DataStream;
+
+DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+ source
+ .fromFile("～/admin/data/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
+```
+
+## Maven Repository
+
+```xml
+
+<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+</dependency>
+```
+
+# Core API
+
+RocketMQ Streams implements a series of advanced APIs, allowing users to write stream computing programs conveniently and achieve their own business requirements.
+
+## StreamBuilder
+
+StreamBuilder is used to build the source of stream tasks. It contains two methods: ```dataStream()``` and ```tableStream()```, which return two sources, DataStreamSource and TableStreamSource, respectively.
+
++ [dataStream(nameSpaceName,pipelineName)]() returns an instance of DataStreamSource, used for segmented programming to achieve stream computing tasks.
++ [tableStream(nameSpaceName,pipelineName)]() returns an instance of TableStreamSource, used for script programming to achieve stream computing tasks.
+
+## DataStream API
+
+### Source
+
+DataStreamSource is a source class of segmented programming, used to interface with various data sources and obtain data from major message queues.
+
++ ```fromFile```: reads data from the file. This method contains two parameters:
+ + ```filePath```: specifies which file path to read. Required.
+ + ```isJsonData```: specifies whether data is in JSON format. Optional. Default value: ```true```.
+ + ```tags```: the tags for filtering messages used by the RocketMQ consumer. Optional.
+
+
++ ```fromRocketmq```: obtains data from RocketMQ, including four parameters:
+ + ```topic```: the topic name of RocketMQ. Required.
+ + ```groupName```: the name of the consumer group. Required.
+ + ```isJson```: specifies whether data is in JSON format. Optional.
+ + ```tags```: the tags for filtering messages used by the RocketMQ consumer. Optional.
+
++ ```from```: custom data source. You can specify your own data source by implementing ISource interface.
+
+### transform
+
+transform allows the input source data to be modified during the stream calculation process for the next step; DataStream API includes ```DataStream```, ```JoinStream```, ```SplitStream```, ```WindowStream```, and many other transform classes.
+
+#### DataStream
+
+DataStream implements a series of common stream calculation operators as follows:
+
++ ```map```: returns a new DataStream by passing each record of the source to the **func** function.
++ ```flatmap```: similar to map. One input item corresponds to 0 or more output items.
++ ```filter```: returns a new DataStream based on the record of the source DataStream only when the ** func** function returns **true**.
++ ```forEach```: executes the **func** function once for each record and returns a new DataStream.
++ ```selectFields```: returns the corresponding field value for each record, and returns a new DataStream.
++ ```operate```: executes a custom function for each record and returns a new DataStream.
++ ```script```: executes a script for each recorded field, returns new fields, and generates a new DataStream.
++ ```toPrint```: prints the result on the console and generates a new DataStreamAction instance.
++ ```toFile```: saves the result as a file and generates a new DataStreamAction instance.
++ ```toDB```: saves the result to the database.
++ ```toRocketmq```: outputs the result to RocketMQ.
++ ```to```: outputs the result to the specified storage through the custom ISink interface.
++ ```window```: performs relevant statistical analysis in the window, generally used in conjunction with ```groupBy```. ```window()``` is used to define the size of the window, and ```groupBy( )``` used to define the main key of statistical analysis. You can specify multiple main keys:
+ + ```count```: counts in the window.
+ + ```min```: gets the minimum of the statistical value in the window.
+ + ```max```: gets the maximum of the statistical value in the window.
+ + ```avg```: gets the average of the statistical values in the window.
+ + ```sum```: gets the sum of the statistical values in the window.
+ + ```reduce```: performs custom summary calculations in the window.
++ ```join```: associates the two streams or one stream and one physical table according to the conditions and merges them into a large stream for related calculations.
+ + ```dimJoin``` associate a stream with a physical table which can be a file or a db table, and all matching records are retained
+ + ```dimLeftJoin``` After a flow is associated with a physical table, all data of the flow is reserved and fields that do not match the physical table are left blank
+ + ```join```
+ + ```leftJoin```
++ ```union```: merges the two streams.
++ ```split```: splits a data stream into different data streams according to tags for downstream analysis and calculation.
++ ```with```: specifies related strategies during the calculation, including Checkpoint and state storage strategies, etc.
+
+# Strategy
+
+The Strategy mechanism is mainly used to control the underlying logic during the operation of the computing engine, such as the storage methods of Checkpoint and state etc. Subsequent controls for windows, dual-stream joins, and so on will be added. All control strategies are transmitted through the ```with``` operator. Multiple policy types can be transmitted at the same time.
+
+```java
+//Specify the storage strategy for Checkpoint.
+source
+ .fromRocketmq("TSG_META_INFO","")
+ .map(message->message+"--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
+ .start();
+```
+
++=======
+ * [Quick Start](quick\_start.md)
+ * [创建实时任务数据源](stream\_source.md)
+ * [创建实时任务数据输出](stream\_sink.md)
+ * [数据处理逻辑](stream\_transform.md)
++>>>>>>> 1cd2dd0291dbcab033e6773021ddca13ce819f82
diff --cc pom.xml
index 5eac49b1,b2792adc..9da26dcd
--- a/pom.xml
+++ b/pom.xml
@@@ -100,12 -52,13 +52,11 @@@
<module>rocketmq-streams-channel-http</module>
<module>rocketmq-streams-state</module>
<module>rocketmq-streams-examples</module>
-- <module>rocketmq-streams-checkpoint</module>
-- <module>rocketmq-streams-connectors</module>
<module>rocketmq-streams-channel-syslog</module>
<module>rocketmq-streams-channel-es</module>
- <module>rocketmq-streams-runner</module>
+ <module>rocketmq-streams-channel-kafka</module>
<module>rocketmq-streams-channel-mqtt</module>
+ <module>rocketmq-streams-cep</module>
</modules>
<properties>
@@@ -119,13 -72,12 +70,13 @@@
<java.version>1.8</java.version>
<java.encoding>UTF-8</java.encoding>
<project.build.sourceEncoding>${java.encoding}</project.build.sourceEncoding>
- <log4j.version>1.2.17</log4j.version>
<commons-logging.version>1.1</commons-logging.version>
- <spring.version>3.2.13.RELEASE</spring.version>
+ <spring.version>5.1.14.RELEASE</spring.version>
<auto-service.version>1.0-rc5</auto-service.version>
<mysql-connector.version>5.1.40</mysql-connector.version>
- <fastjson.version>1.2.25</fastjson.version>
- <quartz.version>2.2.1</quartz.version>
+ <fastjson.version>1.2.9</fastjson.version>
+ <quartz.version>2.3.2</quartz.version>
++
<httpclient.version>4.5.13</httpclient.version>
<commons-io.version>2.7</commons-io.version>
<junit.version>4.13.1</junit.version>
diff --cc rocketmq-streams-channel-es/pom.xml
index 15723b3c,9e872a51..fe50aeef
--- a/rocketmq-streams-channel-es/pom.xml
+++ b/rocketmq-streams-channel-es/pom.xml
@@@ -20,8 -7,8 +7,8 @@@
<parent>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
- </parent>
+ <version>1.0.2-SNAPSHOT</version>
- </parent>
++ </parent>
<artifactId>rocketmq-streams-channel-es</artifactId>
<name>ROCKETMQ STREAMS :: channel-es</name>
<packaging>jar</packaging>
diff --cc rocketmq-streams-channel-mqtt/pom.xml
index 1ebfd008,abf10b72..0141989c
--- a/rocketmq-streams-channel-mqtt/pom.xml
+++ b/rocketmq-streams-channel-mqtt/pom.xml
@@@ -19,8 -5,8 +5,8 @@@
<parent>
<artifactId>rocketmq-streams</artifactId>
<groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
- </parent>
+ <version>1.0.2-SNAPSHOT</version>
- </parent>
++ </parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>rocketmq-streams-channel-mqtt</artifactId>
diff --cc rocketmq-streams-checkpoint/pom.xml
index 6da59e3a,312de599..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-checkpoint/pom.xml
+++ /dev/null
@@@ -1,64 -1,66 +1,0 @@@
--<?xml version="1.0" encoding="UTF-8"?>
--<!--
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- -->
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-- <parent>
-- <artifactId>rocketmq-streams</artifactId>
-- <groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
- <version>1.0.2-SNAPSHOT</version>
-- </parent>
-- <modelVersion>4.0.0</modelVersion>
--
-- <artifactId>rocketmq-streams-checkpoint</artifactId>
-- <name>ROCKETMQ STREAMS :: checkpoint</name>
-- <packaging>jar</packaging>
--
-- <properties>
-- <maven.compiler.source>8</maven.compiler.source>
-- <maven.compiler.target>8</maven.compiler.target>
-- </properties>
-- <dependencies>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-commons</artifactId>
-- <exclusions>
-- <exclusion>
-- <groupId>com.google.auto.service</groupId>
-- <artifactId>auto-service</artifactId>
-- </exclusion>
-- </exclusions>
-- </dependency>
--
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-db-operator</artifactId>
-- <exclusions>
-- <exclusion>
-- <groupId>com.google.auto.service</groupId>
-- <artifactId>auto-service</artifactId>
-- </exclusion>
-- </exclusions>
-- </dependency>
--
-- <dependency>
-- <groupId>com.google.auto.service</groupId>
-- <artifactId>auto-service</artifactId>
-- <optional>true</optional>
-- </dependency>
-- </dependencies>
--
--</project>
diff --cc rocketmq-streams-checkpoint/src/main/java/org/apache/rocketmq/streams/checkpoint/db/DBCheckPointStorage.java
index 69150acf,69150acf..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-checkpoint/src/main/java/org/apache/rocketmq/streams/checkpoint/db/DBCheckPointStorage.java
+++ /dev/null
@@@ -1,68 -1,68 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.checkpoint.db;
--
--import java.util.List;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.common.channel.source.ISource;
--import org.apache.rocketmq.streams.common.checkpoint.AbstractCheckPointStorage;
--import org.apache.rocketmq.streams.common.checkpoint.CheckPoint;
--import org.apache.rocketmq.streams.common.checkpoint.CheckPointManager;
--import org.apache.rocketmq.streams.common.checkpoint.SourceSnapShot;
--import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
--
--/**
-- * @description
-- */
--public class DBCheckPointStorage extends AbstractCheckPointStorage {
--
-- static final Log logger = LogFactory.getLog(DBCheckPointStorage.class);
-- static final String STORAGE_NAME = "DB";
--
-- public DBCheckPointStorage() {
--
-- }
--
-- @Override
-- public String getStorageName() {
-- return STORAGE_NAME;
-- }
--
-- @Override
-- public <T> void save(List<T> checkPointState) {
-- logger.info(String.format("save checkpoint size %d", checkPointState.size()));
-- ORMUtil.batchReplaceInto(checkPointState);
-- }
--
-- @Override
-- public void finish() {
--
-- }
--
-- @Override
-- //todo
-- public CheckPoint recover(ISource iSource, String queueId) {
-- String sourceName = CheckPointManager.createSourceName(iSource, null);
-- String key = CheckPointManager.createCheckPointKey(sourceName, queueId);
-- String sql = "select * from source_snap_shot where `key` = " + "'" + key + "';";
-- SourceSnapShot snapShot = ORMUtil.queryForObject(sql, null, SourceSnapShot.class);
--
-- logger.info(String.format("checkpoint recover key is %s, sql is %s, recover sourceSnapShot : %s", key, sql, snapShot == null ? "null snapShot" : snapShot.toString()));
-- return new CheckPoint().fromSnapShot(snapShot);
-- }
--}
diff --cc rocketmq-streams-connectors/pom.xml
index 8c8277e3,d544e3d5..00000000
deleted file mode 100755,100755
--- a/rocketmq-streams-connectors/pom.xml
+++ /dev/null
@@@ -1,40 -1,47 +1,0 @@@
--<?xml version="1.0" encoding="utf-8"?>
--<!--
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- -->
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-- <modelVersion>4.0.0</modelVersion>
-- <parent>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams</artifactId>
- <version>1.0.2-preview-SNAPSHOT</version>
- <version>1.0.2-SNAPSHOT</version>
-- </parent>
-- <artifactId>rocketmq-streams-connectors</artifactId>
-- <packaging>jar</packaging>
-- <name>ROCKETMQ STREAMS :: connectors</name>
--
-- <dependencies>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-lease</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-schedule</artifactId>
-- </dependency>
--
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-commons</artifactId>
-- </dependency>
-- </dependencies>
--</project>
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/AbstractBalance.java
index 5c2b266f,5c2b266f..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/AbstractBalance.java
+++ /dev/null
@@@ -1,207 -1,207 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.balance;
--
--import java.util.ArrayList;
--import java.util.HashSet;
--import java.util.List;
--import java.util.Set;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.connectors.source.SourceInstance;
--
--public abstract class AbstractBalance implements ISourceBalance {
--
-- protected int balanceCount = 0;
--
-- @Override
-- public SplitChanged doBalance(List<ISplit> allSplits, List<ISplit> ownerSplits) {
-- balanceCount++;
-- heartBeat();
-- List<SourceInstance> sourceInstances = fetchSourceInstances();
-- List<ISplit> workingSplits = fetchWorkingSplits(allSplits);
-- SplitChanged splitChanged = getAdditionSplits(allSplits, sourceInstances, workingSplits, ownerSplits);
-- if (splitChanged != null) {
-- return splitChanged;
-- }
-- splitChanged = getRemoveSplits(allSplits, sourceInstances, workingSplits, ownerSplits);
-- return splitChanged;
-- }
--
-- protected void heartBeat() {
-- holdLockSourceInstance();
-- }
--
-- /**
-- * get all dispatch splits
-- *
-- * @return
-- */
-- protected abstract List<ISplit> fetchWorkingSplits(List<ISplit> allSplitS);
--
-- /**
-- * get all instacne for the source
-- *
-- * @return
-- */
-- protected abstract List<SourceInstance> fetchSourceInstances();
--
-- /**
-- * lock the source ,the lock is globel,only one source instance can get it in same time
-- *
-- * @return
-- */
-- protected abstract boolean holdLockSourceInstance();
--
-- /**
-- * unlock
-- */
-- protected abstract void unlockSourceInstance();
--
-- /**
-- * juge need add split，根据调度策略选择
-- * 每次最大增加的分片数，根据调度次数决定
-- *
-- * @param allSplits
-- * @param sourceInstances
-- * @param workingSplits
-- * @return
-- */
-- protected SplitChanged getAdditionSplits(List<ISplit> allSplits, List<SourceInstance> sourceInstances,
-- List<ISplit> workingSplits, List<ISplit> ownerSplits) {
-- SplitChanged splitChanged = getChangedSplitCount(allSplits, sourceInstances, workingSplits.size(), ownerSplits.size());
-- if (splitChanged == null) {
-- return null;
-- }
-- if (splitChanged.isNewSplit == false) {
-- return null;
-- }
-- if (splitChanged.splitCount <= 0) {
-- return null;
-- }
-- List<ISplit> noWorkingSplits = getNoWorkingSplits(allSplits, workingSplits);
-- List<ISplit> newSplits = new ArrayList<>();
-- for (int i = 0; i < noWorkingSplits.size(); i++) {
-- boolean success = holdLockSplit(noWorkingSplits.get(i));
-- if (success) {
-- newSplits.add(noWorkingSplits.get(i));
-- if (newSplits.size() >= splitChanged.splitCount) {
-- break;
-- }
-- }
-- }
-- splitChanged.setChangedSplits(newSplits);
-- return splitChanged;
--
-- }
--
-- protected List<ISplit> getNoWorkingSplits(List<ISplit> allSplits, List<ISplit> workingSplits) {
-- Set<String> workingSplitIds = new HashSet<>();
-- for (ISplit split : workingSplits) {
-- workingSplitIds.add(split.getQueueId());
-- }
-- List<ISplit> splits = new ArrayList<>();
-- for (ISplit split : allSplits) {
-- if (!workingSplitIds.contains(split.getQueueId())) {
-- splits.add(split);
-- }
-- }
-- return splits;
-- }
--
-- /**
-- * 获取需要删除的分片
-- *
-- * @param allSplits
-- * @param sourceInstances
-- * @param workingSplits
-- * @return
-- */
-- protected SplitChanged getRemoveSplits(List<ISplit> allSplits, List<SourceInstance> sourceInstances,
-- List<ISplit> workingSplits, List<ISplit> ownerSplits) {
-- SplitChanged splitChanged = getChangedSplitCount(allSplits, sourceInstances, workingSplits.size(), ownerSplits.size());
-- if (splitChanged == null) {
-- return null;
-- }
-- if (splitChanged.isNewSplit == true) {
-- return null;
-- }
--
-- if (splitChanged.splitCount <= 0) {
-- return null;
-- }
-- //List<ISplit> ownerSplits=source.ownerSplits();
-- List<ISplit> removeSplits = new ArrayList();
-- for (int i = 0; i < splitChanged.splitCount; i++) {
-- removeSplits.add(ownerSplits.get(i));
-- }
-- splitChanged.setChangedSplits(removeSplits);
-- return splitChanged;
-- }
--
-- /**
-- * 获取需要变动的分片个数，新增或删除
-- * 分配策略，只有有未分配的分片时才会分配新分片，为了减少分片切换，前面几次尽可能少分，后面越来越多
-- *
-- * @return 需要本实例有变化的分配，新增或删除
-- */
-- protected SplitChanged getChangedSplitCount(List<ISplit> allSplits, List<SourceInstance> sourceInstances,
-- int splitCountInWorking, int ownerSplitCount) {
-- //int ownerSplitCount=source.ownerSplits().size();
-- int instanceCount = sourceInstances.size();
-- if (instanceCount == 0) {
-- instanceCount = 1;
-- }
-- int allSplitCount = allSplits.size();
-- int minSplitCount = allSplitCount / instanceCount;
-- int maxSplitCount = minSplitCount + (allSplitCount % instanceCount == 0 ? 0 : 1);
-- //已经是最大分片数了
-- if (ownerSplitCount == maxSplitCount) {
-- return null;
-- }
-- if (ownerSplitCount > maxSplitCount) {
-- int changeSplitCount = ownerSplitCount - maxSplitCount;
-- return new SplitChanged(changeSplitCount, false);
-- }
-- //分片已经全部在处理，当前分片也符合最小分片分配策略，不需要重新分配
-- if (splitCountInWorking == allSplitCount && ownerSplitCount >= minSplitCount) {
-- return null;
-- }
-- //如果还有未分配的分片，且当前实例还有分片的可行性，则分配分片
-- if (splitCountInWorking < allSplitCount && ownerSplitCount < maxSplitCount) {
-- int changeSplitCount = Math.min(maxSplitCount - ownerSplitCount, getMaxSplitCountInOneBalance());
--
-- return new SplitChanged(changeSplitCount, true);
-- }
-- return null;
-- }
--
-- @Override
-- public int getBalanceCount() {
-- return balanceCount;
-- }
--
-- /**
-- * 每次负载均衡最大的分片个数,目的是前几次，少分配分配，可能有实例在启动中，以免频繁切换分片，到后面实例都启动了，斤可能多分配分片
-- *
-- * @return
-- */
-- private int getMaxSplitCountInOneBalance() {
-- int balanceCount = getBalanceCount();
-- return (int) Math.pow(2, balanceCount - 1);
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/ISourceBalance.java
index b012b323,b012b323..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/ISourceBalance.java
+++ /dev/null
@@@ -1,60 -1,60 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.balance;
--
--import java.util.List;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--
--public interface ISourceBalance {
--
-- /**
-- * 做负载均衡
--
-- * @return
-- */
-- SplitChanged doBalance(List<ISplit> allSplits, List<ISplit> ownerSplits);
--
-- /**
-- * 从启动开始，做了多少次均衡
-- * @return
-- */
-- int getBalanceCount();
--
--
--
-- boolean getRemoveSplitLock();
--
-- void unLockRemoveSplitLock();
--
-- /**
-- * lock the split and hold it util the instance is shutdown or remove split
-- * @param split
-- * @return
-- */
-- boolean holdLockSplit(ISplit split);
--
-- /**
-- * unlock split lock
-- * @param split
-- */
-- void unlockSplit(ISplit split);
--
--
-- void setSourceIdentification(String sourceIdentification);
--
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/SplitChanged.java
index c01c1519,c01c1519..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/SplitChanged.java
+++ /dev/null
@@@ -1,55 -1,55 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.balance;
--
--import java.util.List;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--
--public class SplitChanged {
--
-- protected int splitCount;//变动多分片个数
-- protected boolean isNewSplit;//是否新增，false是删除
-- protected List<ISplit> changedSplits;
-- public SplitChanged(int splitCount,boolean isNewSplit){
-- this.splitCount=splitCount;
-- this.isNewSplit=isNewSplit;
-- }
--
-- public int getSplitCount() {
-- return splitCount;
-- }
--
-- public void setSplitCount(int splitCount) {
-- this.splitCount = splitCount;
-- }
--
-- public boolean isNewSplit() {
-- return isNewSplit;
-- }
--
-- public void setNewSplit(boolean newSplit) {
-- isNewSplit = newSplit;
-- }
--
-- public List<ISplit> getChangedSplits() {
-- return changedSplits;
-- }
--
-- public void setChangedSplits(List<ISplit> changedSplits) {
-- this.changedSplits = changedSplits;
-- }
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/impl/LeaseBalanceImpl.java
index dc504e5d,dc504e5d..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/balance/impl/LeaseBalanceImpl.java
+++ /dev/null
@@@ -1,144 -1,144 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.balance.impl;
--
--import com.google.auto.service.AutoService;
--import java.util.ArrayList;
--import java.util.Arrays;
--import java.util.HashMap;
--import java.util.List;
--import java.util.Map;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.common.model.ServiceName;
--import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
--import org.apache.rocketmq.streams.common.utils.RuntimeUtil;
--import org.apache.rocketmq.streams.connectors.balance.AbstractBalance;
--import org.apache.rocketmq.streams.connectors.balance.ISourceBalance;
--import org.apache.rocketmq.streams.connectors.source.SourceInstance;
--import org.apache.rocketmq.streams.lease.LeaseComponent;
--import org.apache.rocketmq.streams.lease.model.LeaseInfo;
--import org.apache.rocketmq.streams.lease.service.ILeaseService;
--
--@AutoService(ISourceBalance.class)
--@ServiceName(LeaseBalanceImpl.DB_BALANCE_NAME)
--public class LeaseBalanceImpl extends AbstractBalance {
--
-- private static final Log logger = LogFactory.getLog(LeaseBalanceImpl.class);
--
-- public static final String DB_BALANCE_NAME = "db_balance";
-- private static final String REMOVE_SPLIT_LOCK_NAME = "lock_remove_split";
-- private static final String SOURCE_LOCK_PREFIX = "SOURCE_";
-- private static final String SPLIT_LOCK_PREFIX = "SPLIT_";
-- protected transient LeaseComponent leaseComponent = LeaseComponent.getInstance();
-- protected transient String sourceIdentification;
--
-- protected int lockTimeSecond = 5;
--
-- public LeaseBalanceImpl(String sourceIdentification) {
--
-- this.sourceIdentification = sourceIdentification;
-- }
--
-- public LeaseBalanceImpl() {
--
-- }
--
-- @Override
-- protected List<ISplit> fetchWorkingSplits(List<ISplit> allSplits) {
-- List<LeaseInfo> leaseInfos = leaseComponent.getService().queryLockedInstanceByNamePrefix(SPLIT_LOCK_PREFIX + this.sourceIdentification, null);
-- logger.info(String.format("lease SPLIT_LOCK_PREFIX is %s, sourceIdentification is %s. ", SPLIT_LOCK_PREFIX, sourceIdentification));
-- if (leaseInfos == null) {
-- return new ArrayList<>();
-- }
--
-- Map<String, ISplit> allSplitMap = new HashMap<>();
-- for (ISplit split : allSplits) {
-- allSplitMap.put(split.getQueueId(), split);
-- }
-- List<ISplit> splits = new ArrayList<>();
-- for (LeaseInfo leaseInfo : leaseInfos) {
-- String leaseName = leaseInfo.getLeaseName();
-- String splitId = MapKeyUtil.getLast(leaseName);
-- splits.add(allSplitMap.get(splitId));
-- }
-- logger.info(String.format("working split is %s", Arrays.toString(splits.toArray())));
-- return splits;
-- }
--
-- @Override
-- protected List<SourceInstance> fetchSourceInstances() {
-- List<LeaseInfo> leaseInfos = leaseComponent.getService().queryLockedInstanceByNamePrefix(SOURCE_LOCK_PREFIX + sourceIdentification, null);
-- if (leaseInfos == null) {
-- return new ArrayList<>();
-- }
-- List<SourceInstance> sourceInstances = new ArrayList<>();
-- for (LeaseInfo leaseInfo : leaseInfos) {
-- String leaseName = leaseInfo.getLeaseName();
-- sourceInstances.add(new SourceInstance(leaseName));
-- }
-- return sourceInstances;
-- }
--
-- @Override
-- protected boolean holdLockSourceInstance() {
-- return holdLock(SOURCE_LOCK_PREFIX + sourceIdentification, RuntimeUtil.getDipperInstanceId());
-- }
--
-- @Override
-- protected void unlockSourceInstance() {
-- leaseComponent.getService().unlock(SOURCE_LOCK_PREFIX + sourceIdentification, RuntimeUtil.getDipperInstanceId());
-- }
--
-- @Override
-- public boolean holdLockSplit(ISplit split) {
-- return holdLock(SPLIT_LOCK_PREFIX + this.sourceIdentification, split.getQueueId());
-- }
--
-- @Override
-- public void unlockSplit(ISplit split) {
-- leaseComponent.getService().unlock(SPLIT_LOCK_PREFIX + this.sourceIdentification, split.getQueueId());
--
-- }
--
-- @Override
-- public boolean getRemoveSplitLock() {
-- return holdLock(this.sourceIdentification, REMOVE_SPLIT_LOCK_NAME);
-- }
--
-- @Override
-- public void unLockRemoveSplitLock() {
-- leaseComponent.getService().unlock(this.sourceIdentification, REMOVE_SPLIT_LOCK_NAME);
-- }
--
-- public String getSourceIdentification() {
-- return sourceIdentification;
-- }
--
-- @Override
-- public void setSourceIdentification(String sourceIdentification) {
-- this.sourceIdentification = sourceIdentification;
-- }
--
-- protected boolean holdLock(String name, String lockName) {
-- ILeaseService leaseService = leaseComponent.getService();
-- boolean success = leaseService.holdLock(name, lockName, lockTimeSecond);
-- return success;
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/PullMessage.java
index 9bf34803,9bf34803..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/PullMessage.java
+++ /dev/null
@@@ -1,50 -1,50 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.model;
--
--import org.apache.rocketmq.streams.common.context.MessageOffset;
--
--public class PullMessage<T> {
-- protected T message;
-- protected MessageOffset messageOffset;
--
-- public T getMessage() {
-- return message;
-- }
--
-- public void setMessage(T message) {
-- this.message = message;
-- }
--
-- public MessageOffset getMessageOffset() {
-- return messageOffset;
-- }
--
-- public void setMessageOffset(MessageOffset messageOffset) {
-- this.messageOffset = messageOffset;
-- }
-- /**
-- * 获取offset字符串，通过.把主offset和子offset串接在一起
-- * @return
-- */
-- public String getOffsetStr(){
-- return this.messageOffset.getOffsetStr();
-- }
-- public String getMainOffset() {
-- return messageOffset.getMainOffset();
-- }
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/ReaderStatus.java
index a4889b5f,a4889b5f..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/model/ReaderStatus.java
+++ /dev/null
@@@ -1,120 -1,120 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.model;
--
--import java.util.Date;
--import java.util.List;
--import org.apache.rocketmq.streams.common.model.Entity;
--import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
--
--/**
-- * @description
-- */
--public class ReaderStatus extends Entity {
--
-- /**
-- * 查询单个readerStatus
-- */
-- static final String queryReaderStatusByUK = "select * from reader_status where source_name = '%s' and reader_name = '%s' and is_finished = 1";
--
-- static final String queryReaderStatusList = "select * from reader_status where source_name = '%s' and is_finished = 1";
--
-- static final String clearReaderStatus = "update reader_status set gmt_modified = now(), is_finished = -1 where source_name = '%s' and reader_name = '%s'";
--
-- String sourceName;
--
-- String readerName;
--
-- int isFinished;
--
-- int totalReader;
--
-- public String getReaderName() {
-- return readerName;
-- }
--
-- public void setReaderName(String readerName) {
-- this.readerName = readerName;
-- }
--
-- public int getIsFinished() {
-- return isFinished;
-- }
--
-- public void setIsFinished(int isFinished) {
-- this.isFinished = isFinished;
-- }
--
-- public int getTotalReader() {
-- return totalReader;
-- }
--
-- public void setTotalReader(int totalReader) {
-- this.totalReader = totalReader;
-- }
--
-- public String getSourceName() {
-- return sourceName;
-- }
--
-- public void setSourceName(String sourceName) {
-- this.sourceName = sourceName;
-- }
--
-- @Override
-- public String toString() {
-- return "ReaderStatus{" +
-- "id=" + id +
-- ", gmtCreate=" + gmtCreate +
-- ", gmtModified=" + gmtModified +
-- ", sourceName='" + sourceName + '\'' +
-- ", readerName='" + readerName + '\'' +
-- ", isFinished=" + isFinished +
-- ", totalReader=" + totalReader +
-- '}';
-- }
--
-- public static ReaderStatus queryReaderStatusByUK(String sourceName, String readerName) {
-- String sql = String.format(queryReaderStatusByUK, sourceName, readerName);
-- ReaderStatus readerStatus = ORMUtil.queryForObject(sql, null, ReaderStatus.class);
-- return readerStatus;
-- }
--
-- public static List<ReaderStatus> queryReaderStatusListBySourceName(String sourceName) {
-- String sql = String.format(queryReaderStatusList, sourceName);
-- List<ReaderStatus> readerStatusList = ORMUtil.queryForList(sql, null, ReaderStatus.class);
-- return readerStatusList;
-- }
--
-- public static void clearReaderStatus(String sourceName, String readerName) {
-- String sql = String.format(clearReaderStatus, sourceName, readerName);
-- ORMUtil.executeSQL(sql, null);
-- }
--
-- public static ReaderStatus create(String sourceName, String readerName, int isFinished, int totalReader) {
--
-- ReaderStatus readerStatus = new ReaderStatus();
-- readerStatus.setSourceName(sourceName);
-- readerStatus.setReaderName(readerName);
-- readerStatus.setIsFinished(isFinished);
-- readerStatus.setTotalReader(totalReader);
-- readerStatus.setGmtCreate(new Date());
-- readerStatus.setGmtModified(new Date());
-- return readerStatus;
--
-- }
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/DBScanReader.java
index 268e891e,268e891e..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/DBScanReader.java
+++ /dev/null
@@@ -1,269 -1,269 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.reader;
--
--import com.alibaba.fastjson.JSON;
--import com.alibaba.fastjson.JSONObject;
--import java.io.Serializable;
--import java.util.ArrayList;
--import java.util.List;
--import java.util.Map;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.common.channel.source.ISource;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.common.component.AbstractComponent;
--import org.apache.rocketmq.streams.common.context.MessageOffset;
--import org.apache.rocketmq.streams.common.utils.ThreadUtil;
--import org.apache.rocketmq.streams.connectors.IBoundedSource;
--import org.apache.rocketmq.streams.connectors.IBoundedSourceReader;
--import org.apache.rocketmq.streams.connectors.model.PullMessage;
--import org.apache.rocketmq.streams.connectors.model.ReaderStatus;
--import org.apache.rocketmq.streams.connectors.source.CycleDynamicMultipleDBScanSource;
--import org.apache.rocketmq.streams.db.driver.DriverBuilder;
--import org.apache.rocketmq.streams.db.driver.JDBCDriver;
--import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
--
--/**
-- * @description
-- */
--public class DBScanReader implements ISplitReader, IBoundedSourceReader, Serializable {
--
-- private static final long serialVersionUID = 8172403250050893288L;
-- private static final Log logger = LogFactory.getLog(DBScanReader.class);
-- static final String sqlTemplate = "select * from %s where id >= %d and id < %d";
--
-- //是否完成了source的call back调用
-- transient volatile boolean isFinishedCall = false;
-- ISource iSource;
-- String url;
-- String userName;
-- String password;
-- String tableName;
-- int batchSize;
-- long offset;
-- long offsetStart;
-- long offsetEnd;
-- long maxOffset;
-- long minOffset;
-- ISplit iSplit;
-- transient List<PullMessage> pullMessages;
-- volatile boolean interrupt = false;
-- volatile boolean isClosed = false;
--
-- public String getUrl() {
-- return url;
-- }
--
-- public void setUrl(String url) {
-- this.url = url;
-- }
--
-- public String getUserName() {
-- return userName;
-- }
--
-- public void setUserName(String userName) {
-- this.userName = userName;
-- }
--
-- public String getPassword() {
-- return password;
-- }
--
-- public void setPassword(String password) {
-- this.password = password;
-- }
--
-- public String getTableName() {
-- return tableName;
-- }
--
-- public void setTableName(String tableName) {
-- this.tableName = tableName;
-- }
--
-- public int getBatchSize() {
-- return batchSize;
-- }
--
-- public void setBatchSize(int batchSize) {
-- this.batchSize = batchSize;
-- }
--
-- public ISplit getISplit() {
-- return iSplit;
-- }
--
-- public void setISplit(ISplit iSplit) {
-- this.iSplit = iSplit;
-- }
--
-- public DBScanReader() {
--
-- }
--
-- transient ThreadLocal<JDBCDriver> threadLocal = new ThreadLocal<JDBCDriver>() {
--
-- @Override
-- public JDBCDriver initialValue() {
-- logger.info(String.format("%s initial jdbcDriver. ", Thread.currentThread().getName()));
-- return DriverBuilder.createDriver(AbstractComponent.DEFAULT_JDBC_DRIVER, url, userName, password);
-- }
--
-- };
--
-- @Override
-- public void open(ISplit split) {
-- this.iSplit = split;
-- JDBCDriver jdbcDriver = threadLocal.get();
-- Map<String, Object> range = jdbcDriver.queryOneRow("select min(id) as min_id, max(id) as max_id from " + tableName);
-- minOffset = Long.parseLong(String.valueOf(range.get("min_id")));
-- maxOffset = Long.parseLong(String.valueOf(range.get("max_id")));
-- offsetStart = minOffset;
-- offset = minOffset;
-- logger.info(String.format("table %s min id [ %d ], max id [ %d ]", tableName, minOffset, maxOffset));
-- pullMessages = new ArrayList<>();
-- }
--
-- @Override
-- public boolean next() {
-- if (interrupt) {
-- return false;
-- }
-- if (isFinished()) {
-- finish();
-- ThreadUtil.sleep(10 * 1000);
-- return false;
-- }
-- JDBCDriver jdbcDriver = threadLocal.get();
-- offsetEnd = offsetStart + batchSize;
-- String batchQuery = String.format(sqlTemplate, tableName, offsetStart, offsetEnd);
-- logger.debug(String.format("execute sql : %s", batchQuery));
-- List<Map<String, Object>> resultData = jdbcDriver.queryForList(batchQuery);
-- offsetStart = offsetEnd;
-- pullMessages.clear();
-- for (Map<String, Object> r : resultData) {
-- PullMessage msg = new PullMessage();
-- JSONObject data = JSONObject.parseObject(JSON.toJSONString(r));
-- msg.setMessage(data);
-- offset = offset > Long.parseLong(data.getString("id")) ? offset : Long.parseLong(data.getString("id"));
-- msg.setMessageOffset(new MessageOffset(String.valueOf(offset), true));
-- pullMessages.add(msg);
-- }
-- return offsetStart - batchSize <= maxOffset;
-- }
--
-- @Override
-- public List<PullMessage> getMessage() {
--// logger.info(String.format("output messages %d", pullMessages.size()));
-- return pullMessages;
-- }
--
-- @Override
-- public SplitCloseFuture close() {
--// interrupt = true;
-- isClosed = true;
-- threadLocal.remove();
-- pullMessages = null;
-- return new SplitCloseFuture(this, iSplit);
-- }
--
-- @Override
-- public void seek(String cursor) {
-- if (cursor == null || cursor.trim().equals("")) {
-- cursor = "0";
-- }
-- offset = Long.parseLong(cursor);
-- if (offset < minOffset) {
-- offset = minOffset;
-- }
-- offsetStart = offset;
-- logger.info(String.format("split %s seek %d.", iSplit.getQueueId(), offset));
-- }
--
-- @Override
-- public String getProgress() {
-- return String.valueOf(offset);
-- }
--
-- @Override
-- public long getDelay() {
-- return maxOffset - offset;
-- }
--
-- @Override
-- public long getFetchedDelay() {
-- return 0;
-- }
--
-- @Override
-- public boolean isClose() {
-- return isClosed;
-- }
--
-- @Override
-- public ISplit getSplit() {
-- return iSplit;
-- }
--
-- @Override
-- public boolean isInterrupt() {
-- return interrupt;
-- }
--
-- @Override
-- public boolean interrupt() {
-- interrupt = true;
-- return true;
-- }
--
-- @Override
-- public boolean isFinished() {
-- return offsetStart > maxOffset;
-- }
--
-- @Override
-- public void finish() {
-- if (isFinishedCall) {
-- return;
-- }
-- pullMessages = null;
-- updateReaderStatus();
-- IBoundedSource tmp = (IBoundedSource) iSource;
-- tmp.boundedFinishedCallBack(this.iSplit);
-- isFinishedCall = true;
-- }
--
-- public ISource getISource() {
-- return iSource;
-- }
--
-- public void setISource(ISource iSource) {
-- this.iSource = iSource;
-- }
--
-- private final void updateReaderStatus() {
-- String sourceName = CycleDynamicMultipleDBScanSource.createKey(this.getISource());
-- int finish = Integer.valueOf(1);
-- int total = ((CycleDynamicMultipleDBScanSource) iSource).getTotalReader();
-- ReaderStatus readerStatus = ReaderStatus.create(sourceName, iSplit.getQueueId(), finish, total);
-- logger.info(String.format("create reader status %s.", readerStatus));
-- ORMUtil.batchReplaceInto(readerStatus);
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/ISplitReader.java
index 6b377cff,6b377cff..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/ISplitReader.java
+++ /dev/null
@@@ -1,96 -1,96 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.reader;
--
--import java.io.IOException;
--import java.io.Serializable;
--import java.util.List;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.connectors.model.PullMessage;
--
--public interface ISplitReader extends Serializable {
--
-- /**
-- * Open.
-- *
-- * @param split the split
-- * @throws IOException the io exception
-- */
-- void open(ISplit split);
--
-- /**
-- * Next boolean.
-- *
-- * @return the boolean
-- * @throws IOException the io exception
-- * @throws InterruptedException the interrupted exception
-- */
-- boolean next();
--
-- /**
-- * Gets message.
-- *
-- * @return the message
-- */
-- List<PullMessage> getMessage();
--
-- /**
-- * Close.
-- *
-- * @throws IOException the io exception
-- */
-- SplitCloseFuture close();
--
-- /**
-- * Seek.
-- *
-- * @param cursor the cursor
-- * @throws IOException the io exception
-- */
-- void seek(String cursor);
--
-- /**
-- * Gets progress.
-- *
-- * @return the progress
-- * @throws IOException the io exception
-- */
-- String getProgress();
--
-- /**
-- * Get message delay (millseconds)
-- *
-- * @return delay
-- */
-- long getDelay();
--
-- /**
-- * Get message delay (millseconds) from being fetched
-- *
-- * @return delay
-- */
-- long getFetchedDelay();
--
-- boolean isClose();
--
-- ISplit getSplit();
--
-- boolean isInterrupt();
--
-- boolean interrupt();
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/SplitCloseFuture.java
index b28748b8,b28748b8..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/reader/SplitCloseFuture.java
+++ /dev/null
@@@ -1,83 -1,83 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.reader;
--
--import java.util.concurrent.ExecutionException;
--import java.util.concurrent.Future;
--import java.util.concurrent.TimeUnit;
--import java.util.concurrent.TimeoutException;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--
--public class SplitCloseFuture implements Future<Boolean> {
--
-- protected ISplitReader reader;
-- protected ISplit split;
--
-- public SplitCloseFuture(ISplitReader reader, ISplit split) {
-- this.reader = reader;
-- this.split = split;
-- }
--
-- @Override
-- public boolean cancel(boolean mayInterruptIfRunning) {
-- return false;
-- }
--
-- @Override
-- public boolean isCancelled() {
-- return false;
-- }
--
-- @Override
-- public boolean isDone() {
-- return reader.isClose();
-- }
--
-- @Override
-- public Boolean get() throws InterruptedException, ExecutionException {
-- synchronized (reader) {
-- reader.wait();
-- }
-- return reader.isClose();
-- }
--
-- @Override
-- public Boolean get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
-- synchronized (reader) {
-- long time = timeout;
-- if (unit == TimeUnit.SECONDS) {
-- time = time * 1000;
-- } else if (unit == TimeUnit.MINUTES) {
-- time = time * 1000 * 60;
-- } else if (unit == TimeUnit.HOURS) {
-- time = time * 1000 * 60 * 60;
-- } else {
-- throw new RuntimeException("can not support this timeout, expect less hour " + timeout + " the unit is " + unit);
-- }
-- reader.wait(time);
-- }
-- return reader.isClose();
-- }
--
-- public ISplitReader getReader() {
-- return reader;
-- }
--
-- public ISplit getSplit() {
-- return split;
-- }
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/AbstractPullSource.java
index fb09a6bd,9aedc95a..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/AbstractPullSource.java
+++ /dev/null
@@@ -1,272 -1,312 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source;
--
--import com.alibaba.fastjson.JSON;
--import com.alibaba.fastjson.JSONObject;
--import java.util.ArrayList;
-import java.util.Collection;
--import java.util.HashMap;
--import java.util.HashSet;
-import java.util.Iterator;
--import java.util.List;
--import java.util.Map;
--import java.util.Set;
--import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
--import java.util.concurrent.ScheduledExecutorService;
--import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadPoolExecutor;
--import java.util.concurrent.TimeUnit;
--import org.apache.commons.lang3.concurrent.BasicThreadFactory;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.common.channel.source.AbstractSource;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.common.checkpoint.CheckPoint;
--import org.apache.rocketmq.streams.common.checkpoint.CheckPointManager;
--import org.apache.rocketmq.streams.common.context.Message;
-import org.apache.rocketmq.streams.common.threadpool.ThreadPoolFactory;
--import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
--import org.apache.rocketmq.streams.connectors.balance.ISourceBalance;
--import org.apache.rocketmq.streams.connectors.balance.SplitChanged;
--import org.apache.rocketmq.streams.connectors.balance.impl.LeaseBalanceImpl;
--import org.apache.rocketmq.streams.connectors.model.PullMessage;
--import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
--import org.apache.rocketmq.streams.connectors.reader.SplitCloseFuture;
--import org.apache.rocketmq.streams.serviceloader.ServiceLoaderComponent;
--
--public abstract class AbstractPullSource extends AbstractSource implements IPullSource<AbstractSource> {
--
-- private static final Log logger = LogFactory.getLog(AbstractPullSource.class);
--
-- protected transient ISourceBalance balance;// balance interface
-- protected transient ScheduledExecutorService balanceExecutor;//schdeule balance
-- protected transient Map<String, ISplitReader> splitReaders = new HashMap<>();//owner split readers
-- protected transient Map<String, ISplit> ownerSplits = new HashMap<>();//working splits by the source instance
--
-- //可以有多种实现，通过名字选择不同的实现
-- protected String balanceName = LeaseBalanceImpl.DB_BALANCE_NAME;
-- //balance schedule time
-- protected int balanceTimeSecond = 10;
-- protected long pullIntervalMs;
- transient CheckPointManager checkPointManager = new CheckPointManager();
-
- protected transient CheckPointManager checkPointManager = new CheckPointManager();
- protected transient boolean shutDown=false;
-- @Override
-- protected boolean startSource() {
-- ServiceLoaderComponent serviceLoaderComponent = ServiceLoaderComponent.getInstance(ISourceBalance.class);
-- balance = (ISourceBalance) serviceLoaderComponent.getService().loadService(balanceName);
-- balance.setSourceIdentification(MapKeyUtil.createKey(getNameSpace(), getConfigureName()));
-- balanceExecutor = new ScheduledThreadPoolExecutor(1, new BasicThreadFactory.Builder().namingPattern("balance-task-%d").daemon(true).build());
-- List<ISplit> allSplits = fetchAllSplits();
-- SplitChanged splitChanged = balance.doBalance(allSplits, new ArrayList(ownerSplits.values()));
-- doSplitChanged(splitChanged);
-- balanceExecutor.scheduleWithFixedDelay(new Runnable() {
-- @Override
-- public void run() {
-- logger.info("balance running..... current splits is " + ownerSplits);
-- List<ISplit> allSplits = fetchAllSplits();
-- SplitChanged splitChanged = balance.doBalance(allSplits, new ArrayList(ownerSplits.values()));
-- doSplitChanged(splitChanged);
-- }
-- }, balanceTimeSecond, balanceTimeSecond, TimeUnit.SECONDS);
-
- startWorks();
-- return true;
- }
-
- private void startWorks() {
- ExecutorService workThreads= ThreadPoolFactory.createThreadPool(maxThread);
- long start=System.currentTimeMillis();
- while (!shutDown) {
- Iterator<Map.Entry<String, ISplitReader>> it = splitReaders.entrySet().iterator();
- while (it.hasNext()) {
- Map.Entry<String, ISplitReader> entry=it.next();
- String splitId=entry.getKey();
- ISplit split=ownerSplits.get(splitId);
- ISplitReader reader=entry.getValue();
- ReaderRunner runner=new ReaderRunner(split,reader);
- workThreads.execute(runner);
- }
- try {
- long sleepTime=this.pullIntervalMs-(System.currentTimeMillis()-start);
- if(sleepTime>0){
- Thread.sleep(sleepTime);
- }
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-- }
--
-- @Override
-- public Map<String, ISplit> getAllSplitMap() {
-- List<ISplit> splits = fetchAllSplits();
-- if (splits == null) {
-- return new HashMap<>();
-- }
-- Map<String, ISplit> splitMap = new HashMap<>();
-- for (ISplit split : splits) {
-- splitMap.put(split.getQueueId(), split);
-- }
-- return splitMap;
-- }
--
-- protected void doSplitChanged(SplitChanged splitChanged) {
-- if (splitChanged == null) {
-- return;
-- }
-- if (splitChanged.getSplitCount() == 0) {
-- return;
-- }
-- if (splitChanged.isNewSplit()) {
-- doSplitAddition(splitChanged.getChangedSplits());
-- } else {
-- doSplitRelease(splitChanged.getChangedSplits());
-- }
-- }
--
-- protected void doSplitAddition(List<ISplit> changedSplits) {
-- if (changedSplits == null) {
-- return;
-- }
-- Set<String> splitIds = new HashSet<>();
-- for (ISplit split : changedSplits) {
-- splitIds.add(split.getQueueId());
-- }
-- addNewSplit(splitIds);
-- for (ISplit split : changedSplits) {
-- ISplitReader reader = createSplitReader(split);
-- reader.open(split);
-- reader.seek(loadSplitOffset(split));
-- splitReaders.put(split.getQueueId(), reader);
-- this.ownerSplits.put(split.getQueueId(), split);
- logger.info("start next");
- Thread thread = new Thread(new Runnable() {
- long mLastCheckTime = System.currentTimeMillis();
-
- @Override
- public void run() {
- logger.info("start running");
- while (reader.isInterrupt() == false) {
- if (reader.next()) {
- List<PullMessage> messages = reader.getMessage();
- if (messages != null) {
- for (PullMessage pullMessage : messages) {
- String queueId = split.getQueueId();
- String offset = pullMessage.getOffsetStr();
- JSONObject msg = createJson(pullMessage.getMessage());
- Message message = createMessage(msg, queueId, offset, false);
- message.getHeader().setOffsetIsLong(pullMessage.getMessageOffset().isLongOfMainOffset());
- executeMessage(message);
- }
- }
- }
- long curTime = System.currentTimeMillis();
- if (curTime - mLastCheckTime > getCheckpointTime()) {
- sendCheckpoint(reader.getSplit().getQueueId());
- mLastCheckTime = curTime;
- }
- try {
- Thread.sleep(pullIntervalMs);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
-
- }
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- Set<String> removeSplits = new HashSet<>();
- removeSplits.add(reader.getSplit().getQueueId());
- removeSplit(removeSplits);
- balance.unlockSplit(split);
- reader.close();
- synchronized (reader) {
- reader.notifyAll();
- }
-
- }
- });
- thread.setName("reader-task-" + reader.getSplit().getQueueId());
- thread.start();
-// logger.info("start next");
-// Thread thread = new Thread(new Runnable() {
-//
-// thread.setName("reader-task-" + reader.getSplit().getQueueId());
-// thread.start();
-- }
--
-- }
--
-- @Override
-- public String loadSplitOffset(ISplit split) {
-- String offset = null;
-- CheckPoint<String> checkPoint = checkPointManager.recover(this, split);
-- if (checkPoint != null) {
-- offset = JSON.parseObject(checkPoint.getData()).getString("offset");
-- }
-- return offset;
-- }
--
-- protected abstract ISplitReader createSplitReader(ISplit split);
--
-- protected void doSplitRelease(List<ISplit> changedSplits) {
-- boolean success = balance.getRemoveSplitLock();
-- if (!success) {
-- return;
-- }
-- try {
-- List<SplitCloseFuture> closeFutures = new ArrayList<>();
-- for (ISplit split : changedSplits) {
-- ISplitReader reader = this.splitReaders.get(split.getQueueId());
-- if (reader == null) {
-- continue;
-- }
-- SplitCloseFuture future = reader.close();
-- closeFutures.add(future);
-- }
-- for (SplitCloseFuture future : closeFutures) {
-- try {
- future.get();
- if(!future.isDone()){
- future.get();
- }
-- this.splitReaders.remove(future.getSplit().getQueueId());
-- this.ownerSplits.remove(future.getSplit().getQueueId());
-- } catch (InterruptedException e) {
-- e.printStackTrace();
-- } catch (ExecutionException e) {
-- e.printStackTrace();
-- }
-- }
--
-- } finally {
-- balance.unLockRemoveSplitLock();
- }
-
- }
-
-
- protected class ReaderRunner implements Runnable{
- long mLastCheckTime = System.currentTimeMillis();
- protected ISplit split;
- protected ISplitReader reader;
-
- public ReaderRunner(ISplit split,ISplitReader reader){
- this.split=split;
- this.reader=reader;
- }
-
- @Override
- public void run() {
- logger.info("start running");
- if (reader.isInterrupt() == false) {
- if (reader.next()) {
- List<PullMessage> messages = reader.getMessage();
- if (messages != null) {
- for (PullMessage pullMessage : messages) {
- String queueId = split.getQueueId();
- String offset = pullMessage.getOffsetStr();
- JSONObject msg = createJson(pullMessage.getMessage());
- Message message = createMessage(msg, queueId, offset, false);
- message.getHeader().setOffsetIsLong(pullMessage.getMessageOffset().isLongOfMainOffset());
- executeMessage(message);
- }
- }
- reader.notifyAll();
- }
- long curTime = System.currentTimeMillis();
- if (curTime - mLastCheckTime > getCheckpointTime()) {
- sendCheckpoint(reader.getSplit().getQueueId());
- mLastCheckTime = curTime;
- }
-
-
- }else {
- Set<String> removeSplits = new HashSet<>();
- removeSplits.add(reader.getSplit().getQueueId());
- removeSplit(removeSplits);
- balance.unlockSplit(split);
- reader.close();
- synchronized (reader) {
- reader.notifyAll();
- }
- }
-
-- }
--
-- }
--
-- @Override
-- public boolean supportNewSplitFind() {
-- return true;
-- }
--
-- @Override
-- public boolean supportRemoveSplitFind() {
-- return true;
-- }
--
-- @Override
-- public boolean supportOffsetRest() {
-- return true;
-- }
--
-- @Override
-- public Long getPullIntervalMs() {
-- return pullIntervalMs;
-- }
--
-- public String getBalanceName() {
-- return balanceName;
-- }
--
-- public void setBalanceName(String balanceName) {
-- this.balanceName = balanceName;
-- }
--
-- public int getBalanceTimeSecond() {
-- return balanceTimeSecond;
-- }
--
-- public void setBalanceTimeSecond(int balanceTimeSecond) {
-- this.balanceTimeSecond = balanceTimeSecond;
-- }
--
-- public void setPullIntervalMs(long pullIntervalMs) {
-- this.pullIntervalMs = pullIntervalMs;
-- }
--
-- @Override
-- public List<ISplit> ownerSplits() {
-- return new ArrayList(ownerSplits.values());
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/CycleDynamicMultipleDBScanSource.java
index 561b48f2,561b48f2..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/CycleDynamicMultipleDBScanSource.java
+++ /dev/null
@@@ -1,213 -1,213 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source;
--
--import com.alibaba.fastjson.JSONObject;
--import java.io.Serializable;
--import java.util.Arrays;
--import java.util.Iterator;
--import java.util.List;
--import java.util.Map;
--import java.util.concurrent.ConcurrentHashMap;
--import java.util.concurrent.atomic.AtomicInteger;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.common.channel.source.AbstractSource;
--import org.apache.rocketmq.streams.common.channel.source.ISource;
--import org.apache.rocketmq.streams.common.channel.source.systemmsg.ChangeTableNameMessage;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.common.context.Message;
--import org.apache.rocketmq.streams.common.metadata.MetaDataUtils;
--import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
--import org.apache.rocketmq.streams.common.utils.ThreadUtil;
--import org.apache.rocketmq.streams.connectors.IBoundedSource;
--import org.apache.rocketmq.streams.connectors.model.ReaderStatus;
--import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
--import org.apache.rocketmq.streams.connectors.source.filter.CycleSchedule;
--import org.apache.rocketmq.streams.connectors.source.filter.CycleScheduleFilter;
--import org.apache.rocketmq.streams.db.CycleSplit;
--
--/**
-- * @description
-- */
--public class CycleDynamicMultipleDBScanSource extends DynamicMultipleDBScanSource implements IBoundedSource, Serializable {
--
-- private static final long serialVersionUID = 6840988298037061128L;
-- private static final Log logger = LogFactory.getLog(CycleDynamicMultipleDBScanSource.class);
--
-- Map<String, Boolean> initReaderMap = new ConcurrentHashMap<>();
-- CycleSchedule.Cycle cycle;
-- transient AtomicInteger size = new AtomicInteger(0);
--
-- public CycleDynamicMultipleDBScanSource() {
-- super();
-- }
--
-- public CycleDynamicMultipleDBScanSource(CycleSchedule.Cycle cycle) {
-- super();
-- this.cycle = cycle;
-- }
--
-- public AtomicInteger getSize() {
-- return size;
-- }
--
-- public void setSize(AtomicInteger size) {
-- this.size = size;
-- }
--
-- /**
-- * @return
-- */
-- //todo
-- @Override
-- public synchronized List<ISplit> fetchAllSplits() {
--
-- if (this.filter == null) {
-- filter = new CycleScheduleFilter(cycle.getAllPattern());
-- }
--
-- //如果还是当前周期, 已经完成全部分区的加载, 则不在加载
-- if (size.get() == cycle.getCycleCount()) {
-- return splits;
-- }
-- String sourceName = createKey(this);
-- List<String> tableNames = MetaDataUtils.listTableNameByPattern(url, userName, password, logicTableName + "%");
--
-- logger.info(String.format("load all logic table : %s", Arrays.toString(tableNames.toArray())));
-- Iterator<String> it = tableNames.iterator();
-- while (it.hasNext()) {
-- String s = it.next();
-- String suffix = s.replace(logicTableName + "_", "");
-- if (filter.filter(sourceName, logicTableName, suffix)) {
-- logger.info(String.format("filter add %s", s));
-- CycleSplit split = new CycleSplit();
-- split.setLogicTableName(logicTableName);
-- split.setSuffix(suffix);
-- split.setCyclePeriod(cycle.getCycleDateStr());
-- String splitId = split.getQueueId();
-- if (initReaderMap.get(splitId) == null) {
-- initReaderMap.put(splitId, false);
-- splits.add(split);
-- size.incrementAndGet();
-- }
-- } else {
-- logger.info(String.format("filter remove %s", s));
-- it.remove();
-- }
-- }
--
-- this.tableNames = tableNames;
-- return splits;
-- }
--
-- public Map<String, Boolean> getInitReaderMap() {
-- return initReaderMap;
-- }
--
-- public void setInitReaderMap(Map<String, Boolean> initReaderMap) {
-- this.initReaderMap = initReaderMap;
-- }
--
-- @Override
-- public void finish() {
-- super.finish();
-- for (Map.Entry<String, Boolean> entry : initReaderMap.entrySet()) {
-- String key = entry.getKey();
-- Boolean value = entry.getValue();
-- if (value == false) {
-- logger.error(String.format("split[%s] reader is not finish, exit with error. ", key));
-- }
-- }
-- this.initReaderMap.clear();
-- this.initReaderMap = null;
-- splits.clear();
-- splits = null;
-- }
--
-- @Override
-- public boolean isFinished() {
-- List<ReaderStatus> readerStatuses = ReaderStatus.queryReaderStatusListBySourceName(createKey(this));
-- if (readerStatuses == null) {
-- return false;
-- }
-- return readerStatuses.size() == size.get();
-- }
--
-- @Override
-- protected ISplitReader createSplitReader(ISplit iSplit) {
-- return super.createSplitReader(iSplit);
-- }
--
-- private void sendChangeTableNameMessage() {
-- logger.info(String.format("start send change table name message."));
-- ChangeTableNameMessage changeTableNameMessage = new ChangeTableNameMessage();
-- changeTableNameMessage.setScheduleCycle(cycle.getCycleDateStr());
-- Message message = createMessage(new JSONObject(), null, null, false);
-- message.setSystemMessage(changeTableNameMessage);
-- message.getHeader().setSystemMessage(true);
-- executeMessage(message);
-- logger.info(String.format("finish send change table name message."));
-- }
--
-- @Override
-- public synchronized void boundedFinishedCallBack(ISplit iSplit) {
-- this.initReaderMap.put(iSplit.getQueueId(), true);
-- logger.info(String.format("current map is %s, key is %s. ", initReaderMap, iSplit.getQueueId()));
-- if (statusCheckerStart.compareAndSet(false, true)) {
-- Thread thread = new Thread(new Runnable() {
-- @Override
-- public void run() {
-- while (!isFinished()) {
-- ThreadUtil.sleep(3 * 1000);
-- }
-- logger.info(String.format("source will be closed."));
-- sendChangeTableNameMessage(); //下发修改name的消息
-- ThreadUtil.sleep(1 * 1000);
-- finish();
-- }
--
-- });
-- thread.setName(createKey(this) + "_callback");
-- thread.start();
-- }
-- }
--
-- public CycleSchedule.Cycle getCycle() {
-- return cycle;
-- }
--
-- public void setCycle(CycleSchedule.Cycle cycle) {
-- this.cycle = cycle;
-- }
--
-- @Override
-- public String createCheckPointName() {
-- return super.createCheckPointName();
-- }
--
-- public synchronized int getTotalReader() {
-- return size.get();
-- }
--
-- public static String createKey(ISource iSource) {
-- AbstractSource source = (AbstractSource) iSource;
-- CycleSchedule.Cycle cycle = ((CycleDynamicMultipleDBScanSource) iSource).getCycle();
-- return MapKeyUtil.createKey(source.getNameSpace(), source.getGroupName(), source.getConfigureName(), source.getTopic(), cycle.getCycleDateStr());
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/DynamicMultipleDBScanSource.java
index ea2a118b,ea2a118b..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/DynamicMultipleDBScanSource.java
+++ /dev/null
@@@ -1,190 -1,190 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source;
--
--import java.io.Serializable;
--import java.util.ArrayList;
--import java.util.Arrays;
--import java.util.List;
--import java.util.concurrent.atomic.AtomicBoolean;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--import org.apache.rocketmq.streams.common.metadata.MetaDataUtils;
--import org.apache.rocketmq.streams.connectors.reader.DBScanReader;
--import org.apache.rocketmq.streams.connectors.reader.ISplitReader;
--import org.apache.rocketmq.streams.connectors.source.filter.DataFormatPatternFilter;
--import org.apache.rocketmq.streams.connectors.source.filter.PatternFilter;
--import org.apache.rocketmq.streams.db.DynamicMultipleDBSplit;
--
--/**
-- * @description DynamicMultipleDBScanSource
-- */
--public class DynamicMultipleDBScanSource extends AbstractPullSource implements Serializable {
--
-- private static final long serialVersionUID = 3987103552547019739L;
-- private static final Log logger = LogFactory.getLog(DynamicMultipleDBScanSource.class);
-- public static final int DEFAULT_BATCH_SIZE = 50;
-- public static final int MAX_BATCH_SIZE = 100;
--
-- String url;
-- String userName;
-- String password;
-- String logicTableName;
-- String suffix;
-- int batchSize;
-- List<String> tableNames;
-- List<ISplit> splits;
-- transient volatile AtomicBoolean statusCheckerStart = new AtomicBoolean(false);
--
-- //todo
-- transient PatternFilter filter;
--
-- public DynamicMultipleDBScanSource() {
-- splits = new ArrayList<>();
-- }
--
-- @Override
-- protected boolean initConfigurable() {
-- setTopic(logicTableName);
-- return super.initConfigurable();
-- }
--
-- @Override
-- protected boolean isNotDataSplit(String queueId) {
-- return tableNames.contains(queueId);
-- }
--
-- @Override
-- protected ISplitReader createSplitReader(ISplit split) {
--
-- DBScanReader reader = new DBScanReader();
-- reader.setISplit(split);
-- reader.setUrl(url);
-- reader.setUserName(userName);
-- reader.setPassword(password);
-- reader.setTableName(String.valueOf(split.getQueue()));
-- int local = batchSize <= 0 ? DEFAULT_BATCH_SIZE : batchSize;
-- local = local > MAX_BATCH_SIZE ? MAX_BATCH_SIZE : local;
-- reader.setBatchSize(local);
-- reader.setISource(this);
-- logger.info(String.format("create reader for split %s", split.getQueueId()));
-- return reader;
-- }
--
-- @Override
-- public List<ISplit> fetchAllSplits() {
--
-- if (filter == null) {
-- filter = new DataFormatPatternFilter();
-- }
--
--// String sourceName = createKey(this);
--
-- tableNames = MetaDataUtils.listTableNameByPattern(url, userName, password, logicTableName + "%");
--
-- logger.info(String.format("load all logic table : %s", Arrays.toString(tableNames.toArray())));
--
-- for (String s : tableNames) {
-- String suffix = s.replace(logicTableName + "_", "");
-- if (filter.filter(null, logicTableName, suffix)) {
-- logger.info(String.format("filter add %s", s));
-- DynamicMultipleDBSplit split = new DynamicMultipleDBSplit();
-- split.setLogicTableName(logicTableName);
-- split.setSuffix(suffix);
-- splits.add(split);
-- } else {
-- logger.info(String.format("filter remove %s", s));
-- }
--
-- }
-- return splits;
-- }
--
-- public String getUrl() {
-- return url;
-- }
--
-- public void setUrl(String url) {
-- this.url = url;
-- }
--
-- public String getUserName() {
-- return userName;
-- }
--
-- public void setUserName(String userName) {
-- this.userName = userName;
-- }
--
-- public String getPassword() {
-- return password;
-- }
--
-- public void setPassword(String password) {
-- this.password = password;
-- }
--
-- public String getLogicTableName() {
-- return logicTableName;
-- }
--
-- public void setLogicTableName(String logicTableName) {
-- this.logicTableName = logicTableName;
-- }
--
-- public String getSuffix() {
-- return suffix;
-- }
--
-- public void setSuffix(String suffix) {
-- this.suffix = suffix;
-- }
--
-- public int getBatchSize() {
-- return batchSize;
-- }
--
-- public void setBatchSize(int batchSize) {
-- this.batchSize = batchSize;
-- }
--
-- public List<String> getTableNames() {
-- return tableNames;
-- }
--
-- public void setTableNames(List<String> tableNames) {
-- this.tableNames = tableNames;
-- }
--
-- public List<ISplit> getSplits() {
-- return splits;
-- }
--
-- public void setSplits(List<ISplit> splits) {
-- this.splits = splits;
-- }
--
-- public PatternFilter getFilter() {
-- return filter;
-- }
--
-- public void setFilter(PatternFilter filter) {
-- this.filter = filter;
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/IPullSource.java
index 6733911d,6733911d..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/IPullSource.java
+++ /dev/null
@@@ -1,60 -1,60 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source;
--
--import java.util.Collection;
--import java.util.List;
--import java.util.Map;
--import org.apache.rocketmq.streams.common.channel.source.ISource;
--import org.apache.rocketmq.streams.common.channel.split.ISplit;
--
--/**
-- * poll message，need balance
-- */
--public interface IPullSource<T extends ISource> extends ISource<T> {
--
-- /**
-- * 拥有的分片格式
-- *
-- * @return
-- */
-- Collection<ISplit> ownerSplits();
--
-- /**
-- * get all split for the source
-- *
-- * @return
-- */
-- List<ISplit> fetchAllSplits();
--
-- /**
-- * get all split for the source
-- *
-- * @return
-- */
-- Map<String, ISplit> getAllSplitMap();
--
-- Long getPullIntervalMs();
--
-- /**
-- * get cusor from store
-- *
-- * @return
-- */
-- String loadSplitOffset(ISplit split);
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/BoundedPatternFilter.java
index c06de98d,c06de98d..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/BoundedPatternFilter.java
+++ /dev/null
@@@ -1,53 -1,53 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source.filter;
--
--import java.io.Serializable;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--import org.apache.rocketmq.streams.connectors.model.ReaderStatus;
--
--/**
-- * @description 过滤掉已经完成的reader
-- */
--@Deprecated
--public class BoundedPatternFilter extends AbstractPatternFilter implements Serializable {
--
-- static final Log logger = LogFactory.getLog(BoundedPatternFilter.class);
--
-- @Override
-- public boolean filter(String sourceName, String logicTableName, String tableName) {
--
-- ReaderStatus readerStatus = ReaderStatus.queryReaderStatusByUK(sourceName, logicTableName + "_" + tableName);
-- if (readerStatus != null) {
-- logger.info(String.format("filter sourceName %s, logicTableName %s, suffix %s. ", sourceName, logicTableName, tableName));
-- logger.info(String.format("query result %s", readerStatus.toString()));
-- return true;
-- }
-- if (next == null) {
-- return false;
-- }
-- return next.filter(sourceName, logicTableName, tableName);
-- }
--
-- @Override
-- public PatternFilter setNext(PatternFilter filter) {
-- super.setNext(filter);
-- return this;
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePatternFilter.java
index 3a0193f3,3a0193f3..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePatternFilter.java
+++ /dev/null
@@@ -1,173 -1,173 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source.filter;
--
--import java.io.Serializable;
--import java.text.ParseException;
--import java.text.SimpleDateFormat;
--import java.util.ArrayList;
--import java.util.Date;
--import java.util.List;
--
--/**
-- * @description 用来做分区选取
-- */
--public class CyclePatternFilter extends AbstractPatternFilter implements Serializable {
--
-- private static final long serialVersionUID = -5151597286296228754L;
--
-- public static final int INIT_CYCLE_VERSION = 0;
--
-- CyclePeriod cyclePeriod;
--
-- Date curCycleDateTime; //当前调度周期时间
--
-- long cycleId;
--
-- String firstStartTime; //当前最小时间
--
-- List<String> allPatterns;
--
-- String expression;
--
-- boolean isInit;
--
-- //历史数据读取时使用,表示比起当前相差多少个调度周期
-- final long cycleDiff;
--
-- //todo expr解析
-- public CyclePatternFilter(String expr, Date date) throws ParseException {
-- expression = expr;
-- cycleId = INIT_CYCLE_VERSION;
-- cyclePeriod = CyclePeriod.getInstance(expression);
-- curCycleDateTime = calCycleDateTime(date);
-- allPatterns = new ArrayList<>();
-- isInit = true;
-- if(cyclePeriod.isHistory){
-- Date tmp = cyclePeriod.getHisDate();
-- cycleDiff = curCycleDateTime.getTime()/1000 * 1000 - tmp.getTime()/1000*1000;
-- }else{
-- cycleDiff = 0;
-- }
-- }
--
--
-- /**
-- *
-- * @return 返回date格式的调度周期时间
-- */
-- private Date calCycleDateTime(Date date){
-- return cyclePeriod.format(date);
-- }
--
-- private long calCycle(Date date){
-- Date tmp = calCycleDateTime(date);
-- if(tmp.getTime()/1000 == curCycleDateTime.getTime()/1000){
-- return cycleId;
-- }
-- return nextCycle(tmp);
-- }
--
-- private long nextCycle(Date date){
-- curCycleDateTime = date;
-- cycleId++;
-- calAllPattern();
-- return cycleId;
-- }
--
-- private void calAllPattern(){
-- allPatterns.clear();
-- for(int i = 1; i <= cyclePeriod.getCycle(); i++){
-- long d = (curCycleDateTime.getTime()/1000)*1000 - i * cyclePeriod.getInterval() - cycleDiff;
-- String s = cyclePeriod.getDateFormat().format(new Date(d));
-- allPatterns.add(s);
-- }
-- firstStartTime = allPatterns.get(allPatterns.size() - 1);
-- }
--
-- public boolean isNextCycle(Date date){
-- if(isInit){
-- isInit = false;
-- calAllPattern();
-- return true;
-- }
-- long tmp = cycleId;
-- return calCycle(date) > tmp;
-- }
--
-- public List<String> getAllPatterns() {
-- return allPatterns;
-- }
--
-- public long getCycleId() {
-- return cycleId;
-- }
--
-- public Date getCurCycleDateTime(){
-- return curCycleDateTime;
-- }
--
-- public String getCurCycleDateTimeStr(){
-- return cyclePeriod.getDateFormat().format(curCycleDateTime);
-- }
--
-- public long getCycleDiff() {
-- return cycleDiff;
-- }
--
-- public long getCyclePeriodDiff(){
-- return cycleDiff/cyclePeriod.getInterval();
-- }
--
-- public int getCycle(){
-- return cyclePeriod.getCycle();
-- }
--
-- public String getFirstStartTime() {
-- return firstStartTime;
-- }
--
-- @Override
-- public boolean filter(String sourceName, String logicTableName, String tableName) {
-- return allPatterns.contains(tableName);
-- }
--
--
--
-- public static void main(String[] args) throws ParseException {
--
-- CyclePatternFilter cycle = new CyclePatternFilter("yyyyMMddHHmm - 15m", new Date());
-- System.out.println(cycle);
--
-- System.out.println(cycle.filter(null, null, "202109131650"));
-- System.out.println(cycle.filter(null, null, "20210902000000"));
-- System.out.println(cycle.filter(null, null, "20210908000000"));
-- System.out.println(cycle.filter(null, null, "20210910000000"));
-- System.out.println(cycle.filter(null, null, "20210909230000"));
--
-- System.out.println(new SimpleDateFormat("yyyyMMddHH").parse("2021090923"));
-- System.out.println(new SimpleDateFormat("yyyyMMddhhmmss").parse("20210909230000"));
-- System.out.println(new SimpleDateFormat("yyyyMMddHHmmss").parse("20210909100000"));
-- System.out.println(new SimpleDateFormat("yyyyMMddhhmmss").parse("20210909100000"));
--
--
--
--
-- }
--
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePeriod.java
index 4e6cdd6a,4e6cdd6a..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CyclePeriod.java
+++ /dev/null
@@@ -1,222 -1,222 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source.filter;
--
--import java.text.ParseException;
--import java.text.SimpleDateFormat;
--import java.util.Date;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--
--/**
-- * @Description
-- */
--public enum CyclePeriod {
--
-- CYCLE_PERIOD_DATE() {
-- @Override
-- void argsParser(String expr) throws ParseException {
-- super.argsParser(expr);
-- interval = 24 * 3600 * 1000;
-- int length = expr.length();
-- if (length == 8 && checkFormat(expr, PatternFilter.yyyyMMdd)) {
-- format = PatternFilter.yyyyMMdd;
-- } else if (length == 14 && checkFormat(expr, PatternFilter.yyyyMMddHHmmss)) {
-- format = PatternFilter.yyyyMMddHHmmss;
-- } else {
-- throw new RuntimeException(String.format("unsupported format : %s, only support yyyymmdd 、 yyyymmddhhmmss.", expr));
-- }
-- }
--
-- @Override
-- Date format(Date strDate) {
-- Date date = new Date(strDate.getTime());
-- date.setHours(0);
-- date.setMinutes(0);
-- date.setSeconds(0);
-- return date;
-- }
--
-- },
-- CYCLE_PERIOD_HOUR() {
-- @Override
-- void argsParser(String expr) throws ParseException {
-- super.argsParser(expr);
-- interval = 3600 * 1000;
--
-- int length = expr.length();
-- if (length == 10 && checkFormat(expr, PatternFilter.yyyyMMddHH)) {
-- format = PatternFilter.yyyyMMddHH;
-- } else if (length == 14 && checkFormat(expr, PatternFilter.yyyyMMddHHmmss)) {
-- format = PatternFilter.yyyyMMddHHmmss;
-- } else {
-- throw new RuntimeException(String.format("unsupported format : %s, only support yyyymmdd 、 yyyymmddhhmmss.", expr));
-- }
-- }
--
-- @Override
-- Date format(Date strDate) {
-- Date date = new Date(strDate.getTime());
-- date.setMinutes(0);
-- date.setSeconds(0);
-- return date;
-- }
--
-- },
-- CYCLE_PERIOD_MINUTE() {
-- @Override
-- void argsParser(String expr) throws ParseException {
-- super.argsParser(expr);
-- interval = 60 * 1000;
-- int length = expr.length();
-- if (length == 12 && checkFormat(expr, PatternFilter.yyyyMMddHHmm)) {
-- format = PatternFilter.yyyyMMddHHmm;
-- } else if (length == 14 && checkFormat(expr, PatternFilter.yyyyMMddHHmmss)) {
-- format = PatternFilter.yyyyMMddHHmmss;
-- } else {
-- throw new RuntimeException(String.format("unsupported format : %s, only support yyyymmdd 、 yyyymmddhhmmss.", expr));
-- }
-- }
--
-- @Override
-- Date format(Date strDate) {
-- Date date = new Date(strDate.getTime());
-- date.setSeconds(0);
-- return date;
-- }
--
-- };
--
-- boolean isHistory = false;
--
-- long interval;
--
-- int cycle;
--
-- String format;
--
-- String hisDateString;
--
-- static final Log logger = LogFactory.getLog(CyclePeriod.class);
--
-- void argsParser(String expr) throws ParseException {
-- if (expr.matches("^\\d+$")) {
-- isHistory = true;
-- hisDateString = expr;
-- }
-- }
--
-- Date format(Date strDate) {
-- throw new RuntimeException(String.format("unsupported type.", strDate));
-- }
--
-- /**
-- * expr可能是yyyymmdd 或者 20210917
-- *
-- * @param expr
-- * @param format
-- * @return
-- */
-- final boolean checkFormat(String expr, String format) {
--
-- if (!isHistory) {
-- return expr.equalsIgnoreCase(format);
-- }
--
-- try {
-- new SimpleDateFormat(format).parse(expr);
-- return true;
-- } catch (ParseException e) {
-- logger.error(String.format("error format, expr is %s, format is %s.", expr, format));
-- e.printStackTrace();
-- return false;
-- }
-- }
--
-- public Date getHisDate() throws ParseException {
-- return getDateFormat().parse(hisDateString);
-- }
--
-- public SimpleDateFormat getDateFormat() {
-- return new SimpleDateFormat(format);
-- }
--
-- public long getInterval() {
-- return interval;
-- }
--
-- public boolean isHistory() {
-- return isHistory;
-- }
--
-- public void setHistory(boolean history) {
-- isHistory = history;
-- }
--
-- public void setInterval(long interval) {
-- this.interval = interval;
-- }
--
-- public int getCycle() {
-- return cycle;
-- }
--
-- public void setCycle(int cycle) {
-- this.cycle = cycle;
-- }
--
-- public void setFormat(String format) {
-- this.format = format;
-- }
--
-- public String getFormat() {
-- return format;
-- }
--
-- public String getHisDateString() {
-- return hisDateString;
-- }
--
-- public void setHisDateString(String hisDateString) {
-- this.hisDateString = hisDateString;
-- }
--
-- public static CyclePeriod getInstance(String expression) throws ParseException {
--
-- String[] str = expression.split("\\-");
-- assert str.length == 2 : String.format("expression error : %s. ", expression);
-- String expr = str[0].trim();
-- String tmp = str[1].trim().toLowerCase();
-- String cycleStr = tmp.substring(0, tmp.length() - 1);
-- int cycle = Integer.parseInt(cycleStr);
-- CyclePeriod cyclePeriod = null;
-- if (tmp.endsWith("d")) {
-- cyclePeriod = CYCLE_PERIOD_DATE;
-- } else if (tmp.endsWith("h")) {
-- cyclePeriod = CYCLE_PERIOD_HOUR;
-- } else if (tmp.endsWith("m")) {
-- cyclePeriod = CYCLE_PERIOD_MINUTE;
-- } else {
-- new RuntimeException(String.format("unsupported format : %s", expression));
-- }
-- cyclePeriod.argsParser(expr);
-- cyclePeriod.cycle = cycle;
--
-- return cyclePeriod;
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleSchedule.java
index ba9a2797,ba9a2797..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/CycleSchedule.java
+++ /dev/null
@@@ -1,236 -1,236 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source.filter;
--
--import java.io.Serializable;
--import java.text.ParseException;
--import java.util.ArrayList;
--import java.util.Arrays;
--import java.util.Date;
--import java.util.List;
--import java.util.concurrent.atomic.AtomicLong;
--import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
--
--/**
-- * @description 用来做分区选取
-- */
--public class CycleSchedule implements Serializable {
--
-- private static final long serialVersionUID = -5151597286296228754L;
-- public static final int INIT_CYCLE_VERSION = 0;
-- private static CycleSchedule INSTANCE;
-- CyclePeriod cyclePeriod;
-- AtomicLong cycleId = new AtomicLong(0);
-- String expression;
-- boolean isInit;
-- //历史数据读取时使用,表示比起当前相差多少个调度周期
-- final long cycleDiff;
--
-- public CycleSchedule(String expr, Date date) throws ParseException {
-- Date local = subMs(date);
-- expression = expr;
-- cycleId.set(INIT_CYCLE_VERSION);
-- cyclePeriod = CyclePeriod.getInstance(expression);
-- isInit = true;
-- if (cyclePeriod.isHistory) {
-- Date curCycleDateTime = calCycleDateTime(local);
-- Date tmp = subMs(cyclePeriod.getHisDate());
-- cycleDiff = curCycleDateTime.getTime() - tmp.getTime();
-- } else {
-- cycleDiff = 0;
-- }
-- }
--
-- /**
-- * 去掉毫秒时间戳
-- *
-- * @param date
-- * @return
-- */
-- private Date subMs(Date date) {
-- long time = date.getTime() / 1000 * 1000;
-- return new Date(time);
-- }
--
-- /**
-- * @return 返回date格式的调度周期时间
-- */
-- private Date calCycleDateTime(Date date) {
-- return cyclePeriod.format(date);
-- }
--
-- public Cycle nextCycle(Date date) {
-- Date local = subMs(date);
-- local = cyclePeriod.format(local);
-- if (isInit) {
-- isInit = false;
-- } else {
-- cycleId.incrementAndGet();
-- }
-- List<String> ret = calAllPattern(local);
-- Cycle cycle = new Cycle();
-- cycle.setCycleId(cycleId.get());
-- cycle.setAllPattern(ret);
-- cycle.setCycleDateStr(calCycleDateStr(local));
-- cycle.setCycleCount(cyclePeriod.getCycle());
-- cycle.setCurDateStr(cyclePeriod.getDateFormat().format(local));
-- cycle.setCycleDiff(cycleDiff);
-- return cycle;
-- }
--
-- private String calCycleDateStr(Date date) {
-- long d = date.getTime() - cycleDiff;
-- Date d1 = new Date(d);
-- return cyclePeriod.getDateFormat().format(d1);
-- }
--
-- private List<String> calAllPattern(Date date) {
-- List<String> allPatterns = new ArrayList<>();
-- for (int i = 1; i <= cyclePeriod.getCycle(); i++) {
-- long d = date.getTime() - i * cyclePeriod.getInterval() - cycleDiff;
-- String s = cyclePeriod.getDateFormat().format(new Date(d));
-- allPatterns.add(s);
-- }
-- return allPatterns;
-- }
--
-- public CyclePeriod getCyclePeriod() {
-- return cyclePeriod;
-- }
--
-- public void setCyclePeriod(CyclePeriod cyclePeriod) {
-- this.cyclePeriod = cyclePeriod;
-- }
--
-- public AtomicLong getCycleId() {
-- return cycleId;
-- }
--
-- public void setCycleId(AtomicLong cycleId) {
-- this.cycleId = cycleId;
-- }
--
-- public String getExpression() {
-- return expression;
-- }
--
-- public void setExpression(String expression) {
-- this.expression = expression;
-- }
--
-- public boolean isInit() {
-- return isInit;
-- }
--
-- public void setInit(boolean init) {
-- isInit = init;
-- }
--
-- public long getCycleDiff() {
-- return cycleDiff;
-- }
--
-- public static CycleSchedule getInstance(String expr, Date date) {
-- if (INSTANCE == null) {
-- synchronized (CycleSchedule.class) {
-- if (INSTANCE == null) {
-- try {
-- INSTANCE = new CycleSchedule(expr, date);
-- } catch (ParseException e) {
-- e.printStackTrace();
-- }
-- }
-- }
-- }
-- return INSTANCE;
-- }
--
-- public static class Cycle extends BasedConfigurable implements Serializable {
--
-- private static final long serialVersionUID = 4842560538716388622L;
-- Long cycleId;
-- List<String> allPattern;
-- String cycleDateStr;
-- Integer cycleCount;
-- String curDateStr;
-- long cycleDiff;
--
-- public Integer getCycleCount() {
-- return cycleCount;
-- }
--
-- public void setCycleCount(Integer cycleCount) {
-- this.cycleCount = cycleCount;
-- }
--
-- public Cycle() {
-- }
--
-- public Long getCycleId() {
-- return cycleId;
-- }
--
-- public void setCycleId(Long cycleId) {
-- this.cycleId = cycleId;
-- }
--
-- public List<String> getAllPattern() {
-- return allPattern;
-- }
--
-- public void setAllPattern(List<String> allPattern) {
-- this.allPattern = allPattern;
-- }
--
-- public String getCycleDateStr() {
-- return cycleDateStr;
-- }
--
-- public void setCycleDateStr(String cycleDateStr) {
-- this.cycleDateStr = cycleDateStr;
-- }
--
-- public String getCurDateStr() {
-- return curDateStr;
-- }
--
-- public void setCurDateStr(String curDateStr) {
-- this.curDateStr = curDateStr;
-- }
--
-- public long getCycleDiff() {
-- return cycleDiff;
-- }
--
-- public void setCycleDiff(long cycleDiff) {
-- this.cycleDiff = cycleDiff;
-- }
--
-- @Override
-- public String toString() {
-- return "Cycle{" +
-- "cycleId=" + cycleId +
-- ", cycleDateStr='" + cycleDateStr + '\'' +
-- ", cycleCount=" + cycleCount +
-- ", curDateStr='" + curDateStr + '\'' +
-- ", cycleDiff=" + cycleDiff +
-- ", allPattern=" + Arrays.toString(allPattern.toArray()) +
-- '}';
-- }
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/DataFormatPatternFilter.java
index 0cdc0762,0cdc0762..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/DataFormatPatternFilter.java
+++ /dev/null
@@@ -1,106 -1,106 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source.filter;
--
--import java.io.Serializable;
--import java.text.ParseException;
--import java.text.SimpleDateFormat;
--import org.apache.commons.logging.Log;
--import org.apache.commons.logging.LogFactory;
--
--/**
-- * @description
-- */
--public class DataFormatPatternFilter extends AbstractPatternFilter implements Serializable {
--
-- private static final long serialVersionUID = 3604787588465242642L;
--
-- static final Log logger = LogFactory.getLog(DataFormatPatternFilter.class);
--
-- static final String yyyyMMddHHmmss = "yyyyMMddHHmmss";
-- static final String yyyyMMdd = "yyyyMMdd";
-- static final String yyyyMMddHH = "yyyyMMddHH";
--
-- SimpleDateFormat format1 = new SimpleDateFormat(yyyyMMdd);
-- SimpleDateFormat format2 = new SimpleDateFormat(yyyyMMddHH);
-- SimpleDateFormat format3 = new SimpleDateFormat(yyyyMMddHHmmss);
--
-- @Override
-- public boolean filter(String sourceName, String logicTableName, String tableNameSuffix) {
--
-- int len = tableNameSuffix.length();
-- boolean isFilter = false;
--
-- switch (len) {
-- case 8:
-- try {
-- format1.parse(tableNameSuffix);
-- isFilter = true;
-- } catch (ParseException e) {
-- e.printStackTrace();
-- isFilter = false;
-- }
-- break;
-- case 10:
-- try {
-- format2.parse(tableNameSuffix);
-- isFilter = true;
-- } catch (ParseException e) {
-- e.printStackTrace();
-- isFilter = false;
-- }
-- break;
-- case 14:
-- try {
-- format3.parse(tableNameSuffix);
-- isFilter = true;
-- } catch (ParseException e) {
-- e.printStackTrace();
-- isFilter = false;
-- }
-- break;
-- }
--
-- if (isFilter) {
-- logger.info(String.format("filter sourceName %s, logicTableName %s, suffix %s", sourceName, logicTableName, tableNameSuffix));
-- return true;
-- }
-- if (next != null) {
-- return next.filter(sourceName, logicTableName, tableNameSuffix);
-- }
-- return false;
-- }
--
-- @Override
-- public PatternFilter setNext(PatternFilter filter) {
-- super.setNext(filter);
-- return this;
-- }
--
-- public PatternFilter getNext() {
-- return next;
-- }
--
-- public static void main(String[] args) {
-- DataFormatPatternFilter filter = new DataFormatPatternFilter();
--// System.out.println(filter.filter("20200101"));
--// System.out.println(filter.filter("2020010101"));
--// System.out.println(filter.filter("20200101010101"));
--
-- }
--
--}
diff --cc rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/PatternFilter.java
index 42365007,42365007..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-connectors/src/main/java/org/apache/rocketmq/streams/connectors/source/filter/PatternFilter.java
+++ /dev/null
@@@ -1,41 -1,41 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. See the NOTICE file distributed with
-- * this work for additional information regarding copyright ownership.
-- * The ASF licenses this file to You under the Apache License, Version 2.0
-- * (the "License"); you may not use this file except in compliance with
-- * the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.rocketmq.streams.connectors.source.filter;
--
--/**
-- * @description
-- */
--public interface PatternFilter {
--
-- String yyyyMMddHHmmss = "yyyyMMddHHmmss";
-- String yyyyMMdd = "yyyyMMdd";
-- String yyyyMMddHH = "yyyyMMddHH";
-- String yyyyMMddHHmm = "yyyyMMddHHmm";
--
--
-- /**
-- * 根据sourceName和tableName判断是否符合
-- * @param sourceName
-- * @param tableName
-- * @return
-- */
-- boolean filter(String sourceName, String logicTableName, String tableName);
--
-- PatternFilter setNext(PatternFilter filter);
--
--
--}
diff --cc rocketmq-streams-runner/assembly/distribution.xml
index f44e34fe,f44e34fe..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-runner/assembly/distribution.xml
+++ /dev/null
@@@ -1,69 -1,69 +1,0 @@@
--<!--
--Licensed to the Apache Software Foundation (ASF) under one or more
--contributor license agreements. See the NOTICE file distributed with
--this work for additional information regarding copyright ownership.
--The ASF licenses this file to You under the Apache License, Version 2.0
--(the "License"); you may not use this file except in compliance with
--the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
--Unless required by applicable law or agreed to in writing, software
--distributed under the License is distributed on an "AS IS" BASIS,
--WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--See the License for the specific language governing permissions and
--limitations under the License.
---->
--<assembly>
-- <id>distribution</id>
-- <formats>
-- <format>tar.gz</format>
-- </formats>
-- <fileSets>
-- <fileSet>
-- <fileMode>0775</fileMode>
-- <directory>target/rocketmq-streams-${project.version}-standalone/bin</directory>
-- <outputDirectory>bin</outputDirectory>
-- </fileSet>
-- <fileSet>
-- <directory>target/rocketmq-streams-${project.version}-standalone/conf</directory>
-- <outputDirectory>conf</outputDirectory>
-- </fileSet>
-- <fileSet>
-- <directory>target/rocketmq-streams-${project.version}-standalone/jobs</directory>
-- <outputDirectory>jobs</outputDirectory>
-- </fileSet>
-- <fileSet>
-- <fileMode>0775</fileMode>
-- <directory>target/rocketmq-streams-${project.version}-standalone/lib</directory>
-- <outputDirectory>lib</outputDirectory>
-- </fileSet>
-- <fileSet>
-- <fileMode>0775</fileMode>
-- <directory>target/rocketmq-streams-${project.version}-standalone/log</directory>
-- <outputDirectory>log</outputDirectory>
-- </fileSet>
-- </fileSets>
-- <files>
-- <file>
-- <source>target/rocketmq-streams-${project.version}-standalone/LICENSE</source>
-- <outputDirectory/>
-- </file>
-- <file>
-- <source>target/rocketmq-streams-${project.version}-standalone/NOTICE</source>
-- <outputDirectory/>
-- </file>
-- <file>
-- <source>target/rocketmq-streams-${project.version}-standalone/quick_start.md</source>
-- <outputDirectory/>
-- </file>
-- <file>
-- <source>target/rocketmq-streams-${project.version}-standalone/README.md</source>
-- <outputDirectory/>
-- </file>
-- <file>
-- <source>target/rocketmq-streams-${project.version}-standalone/README-chinese.md</source>
-- <outputDirectory/>
-- </file>
-- </files>
--</assembly>
diff --cc rocketmq-streams-runner/assembly/standalone.xml
index 69075a35,69075a35..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-runner/assembly/standalone.xml
+++ /dev/null
@@@ -1,72 -1,72 +1,0 @@@
--<!--
--Licensed to the Apache Software Foundation (ASF) under one or more
--contributor license agreements. See the NOTICE file distributed with
--this work for additional information regarding copyright ownership.
--The ASF licenses this file to You under the Apache License, Version 2.0
--(the "License"); you may not use this file except in compliance with
--the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
--Unless required by applicable law or agreed to in writing, software
--distributed under the License is distributed on an "AS IS" BASIS,
--WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--See the License for the specific language governing permissions and
--limitations under the License.
---->
--<assembly>
-- <id>standalone</id>
-- <formats>
-- <format>dir</format>
-- </formats>
-- <includeBaseDirectory>false</includeBaseDirectory>
--
-- <fileSets>
-- <fileSet>
-- <directory>bin</directory>
-- <outputDirectory>/bin</outputDirectory>
-- <fileMode>0755</fileMode>
-- </fileSet>
-- <fileSet>
-- <directory>src/main/resources</directory>
-- <outputDirectory>/conf</outputDirectory>
-- <fileMode>0755</fileMode>
-- </fileSet>
-- <fileSet>
-- <directory>conf</directory>
-- <outputDirectory>/conf</outputDirectory>
-- <fileMode>0755</fileMode>
-- </fileSet>
-- <fileSet>
-- <directory>jobs</directory>
-- <outputDirectory>/jobs</outputDirectory>
-- <fileMode>0755</fileMode>
-- </fileSet>
-- <fileSet>
-- <directory>log</directory>
-- <outputDirectory>/log</outputDirectory>
-- <fileMode>0755</fileMode>
-- </fileSet>
-- <fileSet>
-- <directory>../</directory>
-- <outputDirectory>/</outputDirectory>
-- <includes>
-- <include>LICENSE</include>
-- <include>NOTICE</include>
-- <include>quick_start.md</include>
-- <include>README.md</include>
-- <include>README-chinese.md</include>
-- </includes>
-- </fileSet>
-- </fileSets>
--
-- <dependencySets>
-- <dependencySet>
-- <useProjectArtifact>true</useProjectArtifact>
-- <outputDirectory>lib</outputDirectory>
-- <!-- 只包含runtime作用域的依赖 -->
-- <scope>runtime</scope>
-- </dependencySet>
-- </dependencySets>
--
--</assembly>
diff --cc rocketmq-streams-runner/bin/start.sh
index 44797c45,44797c45..00000000
deleted file mode 100755,100755
--- a/rocketmq-streams-runner/bin/start.sh
+++ /dev/null
@@@ -1,58 -1,58 +1,0 @@@
--#!/bin/sh
--set -e
--
--PROG_NAME=$0
--MAIN_CLASS=$1
--
--if [ -z "${MAIN_CLASS}" ]; then
-- usage
--fi
--
--usage() {
-- echo "Usage: $PROG_NAME {mainClass or mainClasses splited with comma}"
-- exit 2 # bad usage
--}
--
--
--JVM_CONFIG=$2
--if [ -z "${JVM_CONFIG}" ]; then
-- JVM_CONFIG="-Xms2048m -Xmx2048m -Xss512k"
--fi
--
--ROCKETMQ_STREAMS_HOME=$(cd $(dirname ${BASH_SOURCE[0]})/..; pwd)
--ROCKETMQ_STREAMS_JOBS_DIR=$ROCKETMQ_STREAMS_HOME/jobs
--ROCKETMQ_STREAMS_DEPENDENCIES=$ROCKETMQ_STREAMS_HOME/lib
--ROCKETMQ_STREAMS_LOGS=$ROCKETMQ_STREAMS_HOME/log/catalina.out
--
--if [ -z "${JAVA_HOME:-}" ]; then
-- JAVA="java -server"
--else
-- JAVA="$JAVA_HOME/bin/java -server"
--fi
--
--JAVA_OPTIONS=${JAVA_OPTIONS:-}
--
--JVM_OPTS=()
--if [ ! -z "${JAVA_OPTIONS}" ]; then
-- JVM_OPTS+=("${JAVA_OPTIONS}")
--fi
--if [ ! -z "${JVM_CONFIG}" ]; then
-- JVM_OPTS+=("${JVM_CONFIG}")
--fi
--
--JVM_OPTS+=("-Dlogback.configurationFile=conf/logback.xml")
--
--
--
--# shellcheck disable=SC2039
--# shellcheck disable=SC2206
--array=(${MAIN_CLASS//,/ })
--
--# shellcheck disable=SC2068
--# shellcheck disable=SC2039
--for var in ${array[@]}
--do
-- # shellcheck disable=SC2068
-- # shellcheck disable=SC2039
-- eval exec $JAVA ${JVM_OPTS[@]} -classpath "$ROCKETMQ_STREAMS_JOBS_DIR/*:$ROCKETMQ_STREAMS_DEPENDENCIES/*" $var "&" >>"$ROCKETMQ_STREAMS_LOGS" 2>&1
--done
diff --cc rocketmq-streams-runner/bin/stop.sh
index 5734c926,5734c926..00000000
deleted file mode 100755,100755
--- a/rocketmq-streams-runner/bin/stop.sh
+++ /dev/null
@@@ -1,33 -1,33 +1,0 @@@
--#!/bin/sh
--set -e
--PROG_NAME=$0
--MAIN_CLASS=$1
--
--if [ -z "${MAIN_CLASS}" ]; then
-- usage
--fi
--
--# shellcheck disable=SC2039
--# shellcheck disable=SC2206
--array=(${MAIN_CLASS//,/ })
--
--# shellcheck disable=SC2068
--# shellcheck disable=SC2039
--for var in ${array[@]}
--do
-- STREAM_JOB_PIC="$(ps -ef | grep "$var" | grep -v grep | grep -v "$PROG_NAME" | awk '{print $2}' | sed 's/addr://g')"
-- if [ ! -z "$STREAM_JOB_PIC" ]; then
-- echo $STREAM_JOB_PIC
-- echo "Stop rocketmq-streams job"
-- echo "kill -9 $STREAM_JOB_PIC"
-- kill -9 $STREAM_JOB_PIC
-- echo "Job($MAIN_CLASS) shutdown completed."
-- else
-- echo "Job($MAIN_CLASS) not started."
-- fi
--done
--
--
--
--
--
diff --cc rocketmq-streams-runner/pom.xml
index 6dcd83f4,bcee09c8..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-runner/pom.xml
+++ /dev/null
@@@ -1,80 -1,66 +1,0 @@@
--<?xml version="1.0" encoding="UTF-8"?>
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-- <parent>
-- <artifactId>rocketmq-streams</artifactId>
-- <groupId>org.apache.rocketmq</groupId>
- <version>1.0.2-preview-SNAPSHOT</version>
- <version>1.0.0-SNAPSHOT</version>
-- </parent>
-- <modelVersion>4.0.0</modelVersion>
--
-- <artifactId>rocketmq-streams-runner</artifactId>
-- <name>ROCKETMQ STREAMS :: runner</name>
--
-- <properties>
-- <maven.compiler.source>8</maven.compiler.source>
-- <maven.compiler.target>8</maven.compiler.target>
-- </properties>
-- <dependencies>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-clients</artifactId>
-- </dependency>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-examples</artifactId>
-- </dependency>
-- </dependencies>
--
--
-- <build>
-- <plugins>
-- <plugin>
-- <groupId>org.apache.maven.plugins</groupId>
-- <artifactId>maven-surefire-plugin</artifactId>
-- </plugin>
-- <plugin>
-- <groupId>org.apache.maven.plugins</groupId>
-- <artifactId>maven-failsafe-plugin</artifactId>
-- </plugin>
-- <plugin>
-- <groupId>org.apache.maven.plugins</groupId>
-- <artifactId>maven-assembly-plugin</artifactId>
-- <executions>
-- <execution>
-- <phase>package</phase>
-- <goals>
-- <goal>single</goal>
-- </goals>
-- </execution>
-- </executions>
-- <configuration>
-- <attach>true</attach>
-- <descriptors>
-- <descriptor>assembly/standalone.xml</descriptor>
-- <descriptor>assembly/distribution.xml</descriptor>
-- </descriptors>
-- <finalName>rocketmq-streams-${project.version}</finalName>
-- <outputDirectory>target</outputDirectory>
-- <workDirectory>target/assembly/work</workDirectory>
-- <tarLongFileMode>warn</tarLongFileMode>
-- </configuration>
-- </plugin>
-- </plugins>
-- </build>
--</project>
diff --cc rocketmq-streams-runner/src/main/resources/log4j.xml
index de547783,de547783..00000000
deleted file mode 100644,100644
--- a/rocketmq-streams-runner/src/main/resources/log4j.xml
+++ /dev/null
@@@ -1,51 -1,51 +1,0 @@@
--<?xml version="1.0" encoding="UTF-8"?>
--<!DOCTYPE log4j:configuration PUBLIC "-//log4j/log4j Configuration//EN" "log4j.dtd">
--
--<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
--
--
-- <appender name="console" class="org.apache.log4j.ConsoleAppender">
-- <layout class="org.apache.log4j.PatternLayout">
-- <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss SSS} %-5p %c{1} %m %n"/>
-- </layout>
-- <filter class="org.apache.log4j.varia.LevelRangeFilter">
-- <param name="levelMin" value="INFO"/>
-- <param name="levelMax" value="ERROR"/>
-- <param name="AcceptOnMatch" value="true"/>
-- </filter>
-- </appender>
--
-- <appender name="fileAppender" class="org.apache.log4j.DailyRollingFileAppender">
-- <param name="File" value="${log4j.home}/rocketmq-streams.log"/>
-- <param name="Append" value="true"/>
-- <param name="DatePattern" value="'.'yyyy-MM-dd'.log'"/>
-- <layout class="org.apache.log4j.PatternLayout">
-- <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss SSS} %-5p %c{1} %m %n"/>
-- </layout>
-- <filter class="org.apache.log4j.varia.LevelRangeFilter">
-- <param name="levelMin" value="${log4j.level}"/>
-- <param name="levelMax" value="ERROR"/>
-- <param name="AcceptOnMatch" value="true"/>
-- </filter>
-- </appender>
--
-- <appender name="traceAppender" class="org.apache.log4j.DailyRollingFileAppender">
-- <param name="File" value="${log4j.home}/trace.log"/>
-- <param name="Append" value="true"/>
-- <param name="DatePattern" value="'.'yyyy-MM-dd'.log'"/>
-- <layout class="org.apache.log4j.PatternLayout">
-- <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss SSS} %-5p %c{1} %m %n"/>
-- </layout>
-- </appender>
--
-- <!--<category name="org.apache.rocketmq.streams.common.utils.TraceUtil" additivity="false">-->
-- <!--<level value="INFO"/>-->
-- <!--<appender-ref ref="traceAppender"/>-->
-- <!--</category>-->
--
-- <root>
-- <appender-ref ref="console"/>
-- <!--<appender-ref ref="fileAppender"/>-->
-- </root>
--
--</log4j:configuration>
[rocketmq-streams] 06/16: feat(db) remove db dependency besides channel-db
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
karp pushed a commit to branch snapshot-1.0.4
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 840c956f5b971b195fe83f2194d29ac36068026e
Merge: a5396881 81bf080a
Author: 维章 <un...@gmail.com>
AuthorDate: Thu May 26 10:50:47 2022 +0800
feat(db) remove db dependency besides channel-db
pom.xml | 53 +--
rocketmq-streams-cep/pom.xml | 64 ---
rocketmq-streams-cep/src/test/resources/log4j.xml | 36 --
rocketmq-streams-channel-kafka/pom.xml | 32 --
.../streams/kafka/KafkaChannelBuilder.java | 91 ----
.../apache/rocketmq/streams/kafka/KafkaSplit.java | 55 ---
.../rocketmq/streams/kafka/sink/KafkaSink.java | 200 ---------
.../rocketmq/streams/kafka/source/KafkaSource.java | 238 ----------
.../rocketmq/streams/kafka/KafkaChannelTest.java | 104 -----
.../src/test/resources/log4j.xml | 20 -
rocketmq-streams-clients/pom.xml | 8 -
.../streams/client/ScheduledStreamBuilder.java | 66 ---
.../rocketmq/streams/client/ScheduledTask.java | 69 ---
.../streams/client/source/DataStreamSource.java | 26 --
.../streams/client/transform/DataStream.java | 111 -----
.../streams/client/transform/JoinStream.java | 76 +---
rocketmq-streams-connectors/pom.xml | 47 --
.../streams/connectors/IBoundedSource.java | 32 --
.../streams/connectors/IBoundedSourceReader.java | 26 --
.../streams/connectors/IScheduleCallback.java | 24 -
.../connectors/balance/AbstractBalance.java | 207 ---------
.../streams/connectors/balance/IBalanceTask.java | 24 -
.../streams/connectors/balance/ISourceBalance.java | 60 ---
.../streams/connectors/balance/SplitChanged.java | 55 ---
.../connectors/balance/impl/LeaseBalanceImpl.java | 144 ------
.../streams/connectors/model/PullMessage.java | 50 ---
.../streams/connectors/model/ReaderStatus.java | 120 -----
.../streams/connectors/reader/DBScanReader.java | 269 ------------
.../streams/connectors/reader/ISplitReader.java | 96 ----
.../connectors/reader/SplitCloseFuture.java | 83 ----
.../connectors/source/AbstractPullSource.java | 312 -------------
.../source/CycleDynamicMultipleDBScanSource.java | 213 ---------
.../source/DynamicMultipleDBScanSource.java | 190 --------
.../streams/connectors/source/IPullSource.java | 60 ---
.../connectors/source/MutilBatchTaskSource.java | 158 -------
.../streams/connectors/source/SourceInstance.java | 37 --
.../source/filter/AbstractPatternFilter.java | 38 --
.../source/filter/BoundedPatternFilter.java | 53 ---
.../source/filter/CyclePatternFilter.java | 173 --------
.../connectors/source/filter/CyclePeriod.java | 222 ----------
.../connectors/source/filter/CycleSchedule.java | 236 ----------
.../source/filter/CycleScheduleFilter.java | 37 --
.../source/filter/DataFormatPatternFilter.java | 106 -----
.../connectors/source/filter/PatternFilter.java | 41 --
rocketmq-streams-dim/pom.xml | 53 ---
.../apache/rocketmq/streams/dim/DimComponent.java | 62 ---
.../streams/dim/builder/AbstractDimParser.java | 52 ---
.../streams/dim/builder/DBDimSQLParser.java | 84 ----
.../rocketmq/streams/dim/builder/DimBuilder.java | 94 ----
.../streams/dim/builder/FileDimSQLParser.java | 42 --
.../streams/dim/builder/IDimSQLParser.java | 30 --
.../streams/dim/builder/SQLParserFactory.java | 36 --
.../function/expression/InExpressionResource.java | 78 ----
.../expression/NotInExpressionResource.java | 44 --
.../dim/function/script/IntelligenceFunction.java | 79 ----
.../script/IntelligenceNameListFunction.java | 24 -
.../dim/function/script/NameListFunction.java | 202 ---------
.../rocketmq/streams/dim/index/DimIndex.java | 204 ---------
.../rocketmq/streams/dim/index/IndexExecutor.java | 209 ---------
.../intelligence/AbstractIntelligenceCache.java | 404 -----------------
.../dim/intelligence/AccountIntelligenceCache.java | 76 ----
.../dim/intelligence/DomainIntelligenceCache.java | 82 ----
.../dim/intelligence/IPIntelligenceCache.java | 107 -----
.../dim/intelligence/URLIntelligenceCache.java | 79 ----
.../rocketmq/streams/dim/model/AbstractDim.java | 489 ---------------------
.../streams/dim/model/AbstractProcShareDim.java | 109 -----
.../apache/rocketmq/streams/dim/model/DBDim.java | 160 -------
.../apache/rocketmq/streams/dim/model/FileDim.java | 58 ---
.../rocketmq/streams/dim/model/IDataCache.java | 26 --
.../rocketmq/streams/dim/model/IDimSource.java | 37 --
.../rocketmq/streams/dim/service/IDimService.java | 65 ---
.../streams/dim/service/impl/DimServiceImpl.java | 90 ----
.../com/aliyun/service/ConfigureLoaderTest.java | 34 --
.../com/aliyun/service/ExpressionExecutorTest.java | 79 ----
.../java/com/aliyun/service/JsonParserTest.java | 40 --
.../com/aliyun/service/NameListFunctionTest.java | 99 -----
.../java/com/aliyun/service/TableCompressTest.java | 145 ------
.../examples/join/RocketmqDimJoinExample.java | 33 --
rocketmq-streams-filter/pom.xml | 4 -
.../streams/filter/builder/ExpressionBuilder.java | 2 +-
.../streams/filter/builder/RuleBuilder.java | 27 +-
.../streams/filter/context/RuleContext.java | 55 +--
.../rocketmq/streams/filter/operator/Rule.java | 26 +-
.../streams/filter/operator/action/Action.java | 9 -
rocketmq-streams-lease/pom.xml | 40 --
.../rocketmq/streams/lease/LeaseComponent.java | 102 -----
.../rocketmq/streams/lease/model/LeaseInfo.java | 127 ------
.../streams/lease/service/ILeaseGetCallback.java | 30 --
.../streams/lease/service/ILeaseService.java | 134 ------
.../streams/lease/service/ILeaseStorage.java | 71 ---
.../streams/lease/service/ILeaseStorasge.java | 63 ---
.../lease/service/impl/BasedLesaseImpl.java | 402 -----------------
.../lease/service/impl/LeaseServiceImpl.java | 279 ------------
.../streams/lease/service/impl/MockLeaseImpl.java | 90 ----
.../lease/service/storages/DBLeaseStorage.java | 228 ----------
.../rocketmq/streams/lease/LeaseComponentTest.java | 117 -----
.../src/test/resources/log4j.xml | 36 --
rocketmq-streams-schedule/pom.xml | 58 ---
.../streams/schedule/ScheduleComponent.java | 64 ---
.../schedule/job/ConfigurableExecutorJob.java | 55 ---
.../streams/schedule/service/IScheduleService.java | 89 ----
.../schedule/service/impl/ScheduleServiceImpl.java | 263 -----------
.../stream/schedule/ScheduleComponentTest.java | 62 ---
.../src/test/resources/log4j.xml | 36 --
rocketmq-streams-window/pom.xml | 11 +-
.../window/operator/join/ExpressionMatcher.java | 141 ++++++
.../streams/window/operator/join/JoinWindow.java | 4 +-
.../streams/window/state/impl/WindowValue.java | 2 -
108 files changed, 169 insertions(+), 10325 deletions(-)
diff --cc pom.xml
index 2d139613,069531da..2349b35e
--- a/pom.xml
+++ b/pom.xml
@@@ -33,9 -31,58 +33,8 @@@
<packaging>pom</packaging>
<url>https://rocketmq.apache.org/</url>
-
- <scm>
- <url>git@github.com:apache/rocketmq-streams.git</url>
- <connection>scm:git:git@github.com:apache/rocketmq-streams.git</connection>
- <developerConnection>scm:git:git@github.com:apache/rocketmq-streams.git</developerConnection>
- <tag>1.0.0-preview</tag>
- </scm>
-
- <mailingLists>
- <mailingList>
- <name>Development List</name>
- <subscribe>dev-subscribe@rocketmq.apache.org</subscribe>
- <unsubscribe>dev-unsubscribe@rocketmq.apache.org</unsubscribe>
- <post>dev@rocketmq.apache.org</post>
- </mailingList>
- <mailingList>
- <name>User List</name>
- <subscribe>users-subscribe@rocketmq.apache.org</subscribe>
- <unsubscribe>users-unsubscribe@rocketmq.apache.org</unsubscribe>
- <post>users@rocketmq.apache.org</post>
- </mailingList>
- <mailingList>
- <name>Commits List</name>
- <subscribe>commits-subscribe@rocketmq.apache.org</subscribe>
- <unsubscribe>commits-unsubscribe@rocketmq.apache.org</unsubscribe>
- <post>commits@rocketmq.apache.org</post>
- </mailingList>
- </mailingLists>
-
- <developers>
- <developer>
- <id>Apache RocketMQ</id>
- <name>Apache RocketMQ of ASF</name>
- <url>https://rocketmq.apache.org/</url>
- </developer>
- </developers>
-
- <organization>
- <name>Apache Software Foundation</name>
- <url>http://www.apache.org</url>
- </organization>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
-
<modules>
<module>rocketmq-streams-commons</module>
- <module>rocketmq-streams-dim</module>
<module>rocketmq-streams-transport-minio</module>
<module>rocketmq-streams-script</module>
<module>rocketmq-streams-configurable</module>
@@@ -205,11 -259,11 +199,6 @@@
<artifactId>rocketmq-streams-commons</artifactId>
<version>${project.version}</version>
</dependency>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-channel-es</artifactId>
-- <version>${project.version}</version>
-- </dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-channel-mqtt</artifactId>
@@@ -255,11 -304,11 +239,6 @@@
<artifactId>rocketmq-streams-db-operator</artifactId>
<version>${project.version}</version>
</dependency>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-dim</artifactId>
-- <version>${project.version}</version>
-- </dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-filter</artifactId>
@@@ -290,11 -330,11 +260,6 @@@
</exclusion>
</exclusions>
</dependency>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-streams-script-python</artifactId>
-- <version>${project.version}</version>
-- </dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-serviceloader</artifactId>
@@@ -325,16 -365,7 +290,6 @@@
<artifactId>rocketmq-streams-channel-rocketmq</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-channel-kafka</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-connectors</artifactId>
- <version>${project.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-examples</artifactId>
@@@ -366,12 -397,12 +321,6 @@@
<version>${rocketmq.version}</version>
</dependency>
-- <dependency>
-- <groupId>org.apache.rocketmq</groupId>
-- <artifactId>rocketmq-tools</artifactId>
-- <version>${rocketmq.version}</version>
-- </dependency>
--
<!-- ================================================= -->
<!-- tool library -->
<!-- ================================================= -->
diff --cc rocketmq-streams-clients/pom.xml
index 2d15efce,ff0e4cb0..e24e55f3
--- a/rocketmq-streams-clients/pom.xml
+++ b/rocketmq-streams-clients/pom.xml
@@@ -40,11 -38,6 +40,7 @@@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-channel-mqtt</artifactId>
</dependency>
+
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-streams-channel-kafka</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-streams-channel-db</artifactId>
diff --cc rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/context/RuleContext.java
index 865683c6,89a398e7..55190561
--- a/rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/context/RuleContext.java
+++ b/rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/context/RuleContext.java
@@@ -36,11 -24,8 +24,7 @@@ import org.apache.rocketmq.streams.comm
import org.apache.rocketmq.streams.common.context.AbstractContext;
import org.apache.rocketmq.streams.common.context.IMessage;
import org.apache.rocketmq.streams.common.context.Message;
- import org.apache.rocketmq.streams.common.metadata.MetaData;
- import org.apache.rocketmq.streams.common.metadata.MetaDataAdapter;
import org.apache.rocketmq.streams.common.monitor.IMonitor;
--import org.apache.rocketmq.streams.common.monitor.TopologyFilterMonitor;
- import org.apache.rocketmq.streams.db.driver.JDBCDriver;
import org.apache.rocketmq.streams.filter.function.expression.ExpressionFunction;
import org.apache.rocketmq.streams.filter.operator.Rule;
import org.apache.rocketmq.streams.filter.operator.action.Action;
@@@ -50,6 -34,17 +34,19 @@@ import org.apache.rocketmq.streams.filt
import org.apache.rocketmq.streams.script.function.model.FunctionConfigure;
import org.apache.rocketmq.streams.script.function.service.impl.ScanFunctionService;
+ import java.io.Serializable;
++import java.util.ArrayList;
++import java.util.List;
+ import java.util.Properties;
+ import java.util.Vector;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.ConcurrentMap;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.LinkedBlockingQueue;
+ import java.util.concurrent.ThreadFactory;
+ import java.util.concurrent.ThreadPoolExecutor;
+ import java.util.concurrent.TimeUnit;
+
public class RuleContext extends AbstractContext<Message> implements Serializable {
private static final Log LOG = LogFactory.getLog(RuleContext.class);
diff --cc rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/operator/Rule.java
index 5ae02236,ab7ed69f..5c75df17
--- a/rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/operator/Rule.java
+++ b/rocketmq-streams-filter/src/main/java/org/apache/rocketmq/streams/filter/operator/Rule.java
@@@ -46,9 -44,7 +46,8 @@@ import org.apache.rocketmq.streams.comm
import org.apache.rocketmq.streams.common.topology.model.AbstractRule;
import org.apache.rocketmq.streams.common.topology.stages.FilterChainStage;
import org.apache.rocketmq.streams.common.utils.TraceUtil;
- import org.apache.rocketmq.streams.db.driver.JDBCDriver;
import org.apache.rocketmq.streams.filter.FilterComponent;
+import org.apache.rocketmq.streams.filter.context.RuleContext;
import org.apache.rocketmq.streams.filter.operator.action.Action;
import org.apache.rocketmq.streams.filter.operator.action.impl.SinkAction;
import org.apache.rocketmq.streams.filter.operator.expression.Expression;
diff --cc rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/state/impl/WindowValue.java
index 029ec495,c8534541..7d7285c0
--- a/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/state/impl/WindowValue.java
+++ b/rocketmq-streams-window/src/main/java/org/apache/rocketmq/streams/window/state/impl/WindowValue.java
@@@ -41,12 -38,9 +41,10 @@@ import org.apache.rocketmq.streams.comm
import org.apache.rocketmq.streams.common.utils.Base64Utils;
import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
import org.apache.rocketmq.streams.common.utils.DateUtil;
- import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+import org.apache.rocketmq.streams.common.utils.SerializeUtil;
import org.apache.rocketmq.streams.common.utils.StringUtil;
import org.apache.rocketmq.streams.common.utils.TraceUtil;
- import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
import org.apache.rocketmq.streams.script.context.FunctionContext;
import org.apache.rocketmq.streams.script.operator.impl.AggregationScript;
import org.apache.rocketmq.streams.script.operator.impl.FunctionScript;