You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by ji...@apache.org on 2021/08/02 10:55:13 UTC
[rocketmq-streams] branch main updated (f349dd9 -> 5ef7b1d)
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git.
from f349dd9 Merge pull request #6 from RonzL/main
new ed97ca3 add channel-db module
new 1992a35 add channel-configurable
new 5ef7b1d Merge pull request #5 from cw68ster/main
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../pom.xml | 8 +-
.../streams/db/sink/AbstractMultiTableSink.java | 150 ++++++
.../apache/rocketmq/streams/db/sink/DBSink.java | 239 +++++++++
.../rocketmq/streams/db/sink/DBSinkBuilder.java | 76 +++
.../streams/db/sink/SelfMultiTableSink.java | 53 ++
.../streams/db/sink/SplitBySerialNumber.java | 23 +-
.../streams/db/sink/SplitByTimeMultiTableSink.java | 23 +-
.../streams/db/sink/db/DBWriteOnlyChannelTest.java | 84 +++
.../pom.xml | 8 +-
.../streams/configuable/ConfigurableComponent.java | 189 +++++++
.../streams/configuable/model/Configure.java | 130 +++++
.../service/AbstractConfigurableService.java | 561 +++++++++++++++++++++
.../AbstractSupportParentConfigureService.java | 263 ++++++++++
.../service/ConfigurableServcieType.java | 22 +-
.../service/ConfigurableServiceFactory.java | 58 +++
.../service/impl/FileConfigureService.java | 250 +++++++++
.../impl/FileSupportParentConfigureService.java | 17 +-
.../service/impl/MemoryConfigureService.java | 122 +++++
.../impl/MemorySupportParentConfigureService.java | 20 +-
.../configurable/ConfigurableComponent.java | 188 +++++++
.../streams/configurable/model/Configure.java | 127 +++++
.../service/AbstractConfigurableService.java | 553 ++++++++++++++++++++
.../AbstractSupportParentConfigureService.java | 255 ++++++++++
.../service/ConfigurableServcieType.java | 15 +-
.../service/ConfigurableServiceFactory.java | 59 +++
.../service/impl/FileConfigureService.java | 249 +++++++++
.../impl/FileSupportParentConfigureService.java | 14 +-
.../service/impl/MemoryConfigureService.java | 121 +++++
.../impl/MemorySupportParentConfigureService.java | 18 +-
.../src/main}/resources/log4j.xml | 0
.../configuable/ConfiguableComponentTest.java | 112 ++++
.../streams/configuable/model}/Person.java | 8 +-
.../configurable/ConfigurableComponentTest.java | 108 ++++
.../streams/configurable/model}/Person.java | 2 +-
.../src/test/resources/log4j.xml | 0
35 files changed, 4030 insertions(+), 95 deletions(-)
copy {rocketmq-streams-lease => rocketmq-streams-channel-db}/pom.xml (73%)
create mode 100644 rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/AbstractMultiTableSink.java
create mode 100644 rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSink.java
create mode 100644 rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSinkBuilder.java
create mode 100644 rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SelfMultiTableSink.java
copy rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java => rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitBySerialNumber.java (61%)
copy rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java => rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitByTimeMultiTableSink.java (60%)
create mode 100644 rocketmq-streams-channel-db/src/test/java/org/apache/rocketmq/streams/db/sink/db/DBWriteOnlyChannelTest.java
copy {rocketmq-streams-transport-minio => rocketmq-streams-configurable}/pom.xml (74%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/ConfigurableComponent.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/model/Configure.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractConfigurableService.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractSupportParentConfigureService.java
copy rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServcieType.java (59%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServiceFactory.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileConfigureService.java
copy rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileSupportParentConfigureService.java (68%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemoryConfigureService.java
copy rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemorySupportParentConfigureService.java (64%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/ConfigurableComponent.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/model/Configure.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractConfigurableService.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractSupportParentConfigureService.java
copy rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServcieType.java (59%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServiceFactory.java
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileConfigureService.java
copy rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileSupportParentConfigureService.java (78%)
create mode 100644 rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemoryConfigureService.java
copy rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java => rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemorySupportParentConfigureService.java (74%)
copy {rocketmq-streams-transport-minio/src/test => rocketmq-streams-configurable/src/main}/resources/log4j.xml (100%)
create mode 100644 rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/ConfiguableComponentTest.java
copy {rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db => rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/model}/Person.java (95%)
create mode 100644 rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/ConfigurableComponentTest.java
copy {rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db => rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/model}/Person.java (98%)
copy {rocketmq-streams-transport-minio => rocketmq-streams-configurable}/src/test/resources/log4j.xml (100%)
[rocketmq-streams] 13/15: Merge pull request #1 from
programer-0/develop
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 6d89c9ad60b17bac2d5d706f587ee306862d25a6
Merge: 997f3df 5555b67
Author: Heng Du <du...@apache.org>
AuthorDate: Mon Aug 2 13:30:04 2021 +0800
Merge pull request #1 from programer-0/develop
add lease、dim and client module
.gitignore | 25 ++
README.md | 108 ++++-
pom.xml | 410 +++++++++++++++++++
rocketmq-streams-clients/pom.xml | 46 +++
.../rocketmq/streams/client/DataStreamAction.java | 101 +++++
.../rocketmq/streams/client/StreamBuilder.java | 28 ++
.../streams/client/source/DataStreamSource.java | 76 ++++
.../client/strategy/CheckpointStrategy.java | 69 ++++
.../streams/client/strategy/StateStrategy.java | 37 ++
.../rocketmq/streams/client/strategy/Strategy.java | 25 ++
.../streams/client/transform/DataStream.java | 437 +++++++++++++++++++++
.../streams/client/transform/JoinStream.java | 212 ++++++++++
.../streams/client/transform/SplitStream.java | 61 +++
.../streams/client/transform/WindowStream.java | 210 ++++++++++
.../client/transform/window/HoppingWindow.java | 32 ++
.../client/transform/window/SessionWindow.java | 32 ++
.../streams/client/transform/window/Time.java | 45 +++
.../client/transform/window/TumblingWindow.java | 33 ++
.../client/transform/window/WindowInfo.java | 83 ++++
.../rocketmq/streams/client/DBDriverTest.java | 75 ++++
.../rocketmq/streams/client/DataStreamTest.java | 107 +++++
.../apache/rocketmq/streams/client/FilterTest.java | 49 +++
.../apache/rocketmq/streams/client/JoinTest.java | 89 +++++
.../apache/rocketmq/streams/client/LeaseTest.java | 98 +++++
.../rocketmq/streams/client/ORMUtilTest.java | 172 ++++++++
.../apache/rocketmq/streams/client/SplitTest.java | 86 ++++
.../apache/rocketmq/streams/client/UnionTest.java | 82 ++++
.../apache/rocketmq/streams/client/WindowTest.java | 86 ++++
.../client/windows/AbstractWindowFireModeTest.java | 189 +++++++++
.../streams/client/windows/WindowFromFileTest.java | 158 ++++++++
.../streams/client/windows/WindowFromMetaq.java | 47 +++
.../client/windows/WindowHighAvailabilityTest.java | 131 ++++++
.../src/test/resources/log4j.xml | 36 ++
rocketmq-streams-dim/pom.xml | 47 +++
.../apache/rocketmq/streams/dim/DimComponent.java | 63 +++
.../rocketmq/streams/dim/builder/DimBuilder.java | 94 +++++
.../function/expression/InExpressionResource.java | 80 ++++
.../expression/NotInExpressionResource.java | 45 +++
.../dim/function/script/IntelligenceFunction.java | 81 ++++
.../script/IntelligenceNameListFunction.java | 24 ++
.../dim/function/script/NameListFunction.java | 203 ++++++++++
.../rocketmq/streams/dim/index/DimIndex.java | 319 +++++++++++++++
.../rocketmq/streams/dim/index/IndexExecutor.java | 258 ++++++++++++
.../intelligence/AbstractIntelligenceCache.java | 395 +++++++++++++++++++
.../dim/intelligence/AccountIntelligenceCache.java | 77 ++++
.../dim/intelligence/DomainIntelligenceCache.java | 83 ++++
.../dim/intelligence/IPIntelligenceCache.java | 108 +++++
.../dim/intelligence/URLIntelligenceCache.java | 80 ++++
.../rocketmq/streams/dim/model/AbstractDim.java | 312 +++++++++++++++
.../streams/dim/model/BooleanFieldDBDim.java | 55 +++
.../apache/rocketmq/streams/dim/model/DBDim.java | 140 +++++++
.../rocketmq/streams/dim/service/IDimService.java | 65 +++
.../streams/dim/service/impl/DimServiceImpl.java | 92 +++++
.../com/aliyun/service/ConfigureLoaderTest.java | 37 ++
.../com/aliyun/service/ExpressionExecutorTest.java | 80 ++++
.../java/com/aliyun/service/JsonParserTest.java | 40 ++
.../com/aliyun/service/NameListFunctionTest.java | 90 +++++
.../java/com/aliyun/service/TableCompressTest.java | 26 ++
rocketmq-streams-lease/pom.xml | 25 ++
.../rocketmq/streams/lease/LeaseComponent.java | 103 +++++
.../rocketmq/streams/lease/model/LeaseInfo.java | 127 ++++++
.../streams/lease/service/ILeaseGetCallback.java | 30 ++
.../streams/lease/service/ILeaseService.java | 136 +++++++
.../streams/lease/service/ILeaseStorage.java | 73 ++++
.../streams/lease/service/ILeaseStorasge.java | 63 +++
.../lease/service/impl/BasedLesaseImpl.java | 404 +++++++++++++++++++
.../lease/service/impl/LeaseServiceImpl.java | 275 +++++++++++++
.../streams/lease/service/impl/MockLeaseImpl.java | 95 +++++
.../lease/service/storages/DBLeaseStorage.java | 229 +++++++++++
.../rocketmq/streams/lease/LeaseComponentTest.java | 119 ++++++
.../src/test/resources/log4j.xml | 20 +
71 files changed, 8067 insertions(+), 1 deletion(-)
[rocketmq-streams] 15/15: Merge pull request #5 from cw68ster/main
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 5ef7b1d40a99262078430f691fed2afa8c8352c2
Merge: f349dd9 1992a35
Author: rongtong <ji...@163.com>
AuthorDate: Mon Aug 2 18:55:10 2021 +0800
Merge pull request #5 from cw68ster/main
add module channel-db,channel-configurable
rocketmq-streams-channel-db/pom.xml | 21 +
.../streams/db/sink/AbstractMultiTableSink.java | 150 ++++++
.../apache/rocketmq/streams/db/sink/DBSink.java | 239 +++++++++
.../rocketmq/streams/db/sink/DBSinkBuilder.java | 76 +++
.../streams/db/sink/SelfMultiTableSink.java | 53 ++
.../streams/db/sink/SplitBySerialNumber.java | 36 ++
.../streams/db/sink/SplitByTimeMultiTableSink.java | 36 ++
.../streams/db/sink/db/DBWriteOnlyChannelTest.java | 84 +++
rocketmq-streams-configurable/pom.xml | 21 +
.../streams/configuable/ConfigurableComponent.java | 189 +++++++
.../streams/configuable/model/Configure.java | 130 +++++
.../service/AbstractConfigurableService.java | 561 +++++++++++++++++++++
.../AbstractSupportParentConfigureService.java | 263 ++++++++++
.../service/ConfigurableServcieType.java | 31 ++
.../service/ConfigurableServiceFactory.java | 58 +++
.../service/impl/FileConfigureService.java | 250 +++++++++
.../impl/FileSupportParentConfigureService.java | 38 ++
.../service/impl/MemoryConfigureService.java | 122 +++++
.../impl/MemorySupportParentConfigureService.java | 39 ++
.../configurable/ConfigurableComponent.java | 188 +++++++
.../streams/configurable/model/Configure.java | 127 +++++
.../service/AbstractConfigurableService.java | 553 ++++++++++++++++++++
.../AbstractSupportParentConfigureService.java | 255 ++++++++++
.../service/ConfigurableServcieType.java | 29 ++
.../service/ConfigurableServiceFactory.java | 59 +++
.../service/impl/FileConfigureService.java | 249 +++++++++
.../impl/FileSupportParentConfigureService.java | 37 ++
.../service/impl/MemoryConfigureService.java | 121 +++++
.../impl/MemorySupportParentConfigureService.java | 37 ++
.../src/main/resources/log4j.xml | 20 +
.../configuable/ConfiguableComponentTest.java | 112 ++++
.../rocketmq/streams/configuable/model/Person.java | 110 ++++
.../configurable/ConfigurableComponentTest.java | 108 ++++
.../streams/configurable/model/Person.java | 110 ++++
.../src/test/resources/log4j.xml | 20 +
35 files changed, 4532 insertions(+)
[rocketmq-streams] 05/15: add channel-configurable
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 1992a3514e723f452b59878be8898d855eab2c7f
Author: vv <ze...@alibaba-inc.com>
AuthorDate: Mon Aug 2 12:04:58 2021 +0800
add channel-configurable
---
rocketmq-streams-configurable/pom.xml | 21 +
.../streams/configuable/ConfigurableComponent.java | 189 +++++++
.../streams/configuable/model/Configure.java | 130 +++++
.../service/AbstractConfigurableService.java | 561 +++++++++++++++++++++
.../AbstractSupportParentConfigureService.java | 263 ++++++++++
.../service/ConfigurableServcieType.java | 31 ++
.../service/ConfigurableServiceFactory.java | 58 +++
.../service/impl/FileConfigureService.java | 250 +++++++++
.../impl/FileSupportParentConfigureService.java | 38 ++
.../service/impl/MemoryConfigureService.java | 122 +++++
.../impl/MemorySupportParentConfigureService.java | 39 ++
.../configurable/ConfigurableComponent.java | 188 +++++++
.../streams/configurable/model/Configure.java | 127 +++++
.../service/AbstractConfigurableService.java | 553 ++++++++++++++++++++
.../AbstractSupportParentConfigureService.java | 255 ++++++++++
.../service/ConfigurableServcieType.java | 29 ++
.../service/ConfigurableServiceFactory.java | 59 +++
.../service/impl/FileConfigureService.java | 249 +++++++++
.../impl/FileSupportParentConfigureService.java | 37 ++
.../service/impl/MemoryConfigureService.java | 121 +++++
.../impl/MemorySupportParentConfigureService.java | 37 ++
.../src/main/resources/log4j.xml | 20 +
.../configuable/ConfiguableComponentTest.java | 112 ++++
.../rocketmq/streams/configuable/model/Person.java | 110 ++++
.../configurable/ConfigurableComponentTest.java | 108 ++++
.../streams/configurable/model/Person.java | 110 ++++
.../src/test/resources/log4j.xml | 20 +
27 files changed, 3837 insertions(+)
diff --git a/rocketmq-streams-configurable/pom.xml b/rocketmq-streams-configurable/pom.xml
new file mode 100755
index 0000000..3f837d8
--- /dev/null
+++ b/rocketmq-streams-configurable/pom.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-configurable</artifactId>
+ <name>ROCKETMQ STREAMS :: configurable</name>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-serviceloader</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/ConfigurableComponent.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/ConfigurableComponent.java
new file mode 100644
index 0000000..fbb35ca
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/ConfigurableComponent.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.component.ConfigureDescriptor;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.utils.ConfigurableUtil;
+import org.apache.rocketmq.streams.configuable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.configuable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.configuable.service.ConfigurableServiceFactory;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * 对Configurable对象，做统一的管理，统一查询，插入和更新。 insert/update 把configuabel对象写入存储，支持文件存储（file），内存存储（memory）和db存储（DB）。可以在配置通过这个ConfigureFileKey.CONNECT_TYPE key 配置 query 是基于内存的查询，对象定时load到内存，可以在属性文件通过这个ConfigureFileKey.POLLING_TIME key配置加载周期，单位是秒 新对象加载后生效，已经存在的对象只有updateFlag发生变化才会被替换
+ */
+public class ConfigurableComponent extends AbstractComponent<IConfigurableService>
+ implements IConfigurableService {
+
+ private static final Log LOG = LogFactory.getLog(ConfigurableComponent.class);
+
+ protected volatile IConfigurableService configureService = null;
+
+ protected transient String namespace;
+
+ public ConfigurableComponent() {
+ initConfigurableServiceDescriptor();
+ addConfigureDescriptor(
+ new ConfigureDescriptor(CONNECT_TYPE, false, ConfigurableServcieType.DEFAULT_SERVICE_NAME));
+ }
+
+ public static ConfigurableComponent getInstance(String namespace) {
+ return ComponentCreator.getComponent(namespace, ConfigurableComponent.class);
+ }
+
+ @Override
+ protected boolean initProperties(Properties properties) {
+ try {
+ if (configureService != null) {
+ return true;
+ }
+ this.configureService = ConfigurableServiceFactory.createConfigurableService(properties);
+ return true;
+ } catch (Exception e) {
+ LOG.error("ConfigurableComponent create error,properties= " + properties, e);
+ return false;
+ }
+
+ }
+
+ @Override
+ public boolean startComponent(String namespace) {
+ try {
+ this.namespace = namespace;
+ configureService.initConfigurables(namespace);
+ return true;
+ } catch (Exception e) {
+ LOG.error("ConfigurableComponent init error, namespace is " + namespace, e);
+ return false;
+ }
+
+ }
+
+ /**
+ * 启动测试模式，用内存数据库存储和加载configurable数据
+ */
+ public static void begineTestMode() {
+ System.setProperty(ConfigurableComponent.CONNECT_TYPE, ConfigurableServcieType.MEMORY_SERVICE_NAME);
+ }
+
+ /**
+ * 关闭测试模式，用配置文件中配置的属性加载configuable数据
+ */
+ public static void endTestMode() {
+ System.clearProperty(ConfigurableComponent.CONNECT_TYPE);
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ @Override
+ public IConfigurableService getService() {
+ return configureService;
+ }
+
+ @Override
+ public void initConfigurables(String namespace) {
+ configureService.initConfigurables(namespace);
+ }
+
+ @Override
+ public boolean refreshConfigurable(String namespace) {
+ return configureService.refreshConfigurable(namespace);
+ }
+
+ public void mockConfigurable(String namespace) {
+ refreshConfigurable(namespace);
+
+ }
+
+ @Override
+ public List<IConfigurable> queryConfigurable(String type) {
+ return configureService.queryConfigurable(type);
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> queryConfigurableByType(String type) {
+ return configureService.queryConfigurableByType(type);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String type, String name) {
+ return configureService.queryConfigurableByIdent(type, name);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String identification) {
+ return configureService.queryConfigurableByIdent(identification);
+ }
+
+ @Override
+ public void insert(IConfigurable configurable) {
+ configureService.insert(configurable);
+ ConfigurableUtil.refreshMock(configurable);
+ }
+
+ @Override
+ public void update(IConfigurable configurable) {
+ configureService.update(configurable);
+ }
+
+ @Override
+ public <T> Map<String, T> queryConfigurableMapByType(String type) {
+ return configureService.queryConfigurableMapByType(type);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T queryConfigurable(String configurableType, String name) {
+ return (T)queryConfigurableByIdent(configurableType, name);
+ }
+
+ //protected void insertConfigurable(JSONObject message, IConfigurable configurable) {
+ // ConfigurableUtil.insertConfigurable(message, configurable, this.configureService);
+ //}
+
+ @Override
+ public String getNamespace() {
+ if (AbstractConfigurableService.class.isInstance(configureService)) {
+ return ((AbstractConfigurableService)configureService).getNamespace();
+ }
+ return namespace;
+ }
+
+ @Override
+ public Collection<IConfigurable> findAll() {
+ return configureService.findAll();
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ return configureService.loadConfigurableFromStorage(type);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/model/Configure.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/model/Configure.java
new file mode 100644
index 0000000..121233d
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/model/Configure.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.model;
+
+import org.apache.rocketmq.streams.common.model.Entity;
+
+/**
+ *
+ * configuable如果存储在db，这个是db表的映射对象
+ */
+public class Configure extends Entity {
+
+ private static final long serialVersionUID = 5668017348345235669L;
+
+ private String nameSpace;
+ private String type;
+ private String name;
+ // private String identification;
+ private String jsonValue;
+ private String modifyTime;
+ private String remark;
+ private int openRange;
+
+
+ public static String createTableSQL(String tableName){
+ return "/******************************************/\n"
+ + "/* TableName = dipper_configure */\n"
+ + "/******************************************/\n"
+ + "CREATE TABLE IF NOT EXISTS `"+tableName+"` (\n"
+ + " `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',\n"
+ + " `gmt_create` datetime NOT NULL COMMENT '创建时间',\n"
+ + " `gmt_modified` datetime NOT NULL COMMENT '修改时间',\n"
+ + " `namespace` varchar(32) NOT NULL COMMENT '项目标识',\n"
+ + " `type` varchar(32) NOT NULL COMMENT '配置类型',\n"
+ + " `name` varchar(128) NOT NULL COMMENT '配置名称',\n"
+ + " `json_value` text NOT NULL COMMENT '配置内容',\n"
+ + " `status` tinyint(3) unsigned NOT NULL DEFAULT '0' COMMENT '1:正在使用 0:已失效',\n"
+ + " PRIMARY KEY (`id`),\n"
+ + " UNIQUE KEY `uk_namespace_type_name` (`namespace`,`type`,`name`),\n"
+ + " KEY `idx_namespace` (`namespace`)\n"
+ + ") ENGINE=InnoDB AUTO_INCREMENT=1814834 DEFAULT CHARSET=utf8 COMMENT='统一接入配置项'\n"
+ + ";";
+ }
+
+ public String getNameSpace() {
+ return nameSpace;
+ }
+
+ public void setNameSpace(String nameSpace) {
+ this.nameSpace = nameSpace;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ // public String getIdentification() {
+ // return identification;
+ // }
+
+ // public void createIdentification() {
+ // this.identification = MapKeyUtil.createKey(nameSpace, type, name);
+ // }
+
+ public String getJsonValue() {
+ return jsonValue;
+ }
+
+ public void setJsonValue(String jsonValue) {
+ this.jsonValue = jsonValue;
+ }
+
+ public String getModifyTime() {
+ return modifyTime;
+ }
+
+ public void setModifyTime(String modifyTime) {
+ this.modifyTime = modifyTime;
+ }
+
+ public String getRemark() {
+ return remark;
+ }
+
+ public void setRemark(String remark) {
+ this.remark = remark;
+ }
+
+ public int getOpenRange() {
+ return openRange;
+ }
+
+ public void setOpenRange(int openRange) {
+ this.openRange = openRange;
+ }
+
+ @Override
+ public String toString() {
+ return "Configure{" + "nameSpace='" + nameSpace + '\'' + ", type='" + type + '\'' + ", name='" + name + '\''
+ + ", jsonValue='" + jsonValue + '\'' + ", modifyTime='" + modifyTime + '\'' + ", remark='" + remark + '\''
+ + ", openRange=" + openRange + '}';
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractConfigurableService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractConfigurableService.java
new file mode 100644
index 0000000..9f55da8
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractConfigurableService.java
@@ -0,0 +1,561 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service;
+
+import com.alibaba.fastjson.JSONObject;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.datatype.DataType;
+import org.apache.rocketmq.streams.common.model.Entity;
+import org.apache.rocketmq.streams.common.utils.ConfigurableUtil;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.configuable.model.Configure;
+
+public abstract class AbstractConfigurableService implements IConfigurableService {
+
+ private static final Log LOG = LogFactory.getLog(AbstractConfigurableService.class);
+
+ private static final String CLASS_NAME = IConfigurableService.CLASS_NAME;
+
+ protected Map<String, List<IConfigurable>> type2ConfigurableMap = new HashMap<>();
+
+ protected Map<String, IConfigurable> name2ConfigurableMap = new HashMap<>();
+
+ protected Map<String, IConfigurable> configurableMap = new HashMap<>();
+
+ protected Properties properties;
+
+ protected transient String namespace;
+
+ public AbstractConfigurableService(Properties properties) {
+ this.properties = properties;
+ }
+
+ public AbstractConfigurableService() {
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String identification) {
+ return name2ConfigurableMap.get(identification);
+ }
+
+ protected String getConfigureKey(String nameSpace, String type, String name) {
+ return MapKeyUtil.createKey(nameSpace, type, name);
+ }
+
+ protected void updateConfiguresCache(IConfigurable configurable) {
+ if (configurable == null) {
+ return;
+ }
+ configurable.toJson();
+ String key = getConfigureKey(configurable.getNameSpace(), configurable.getType(), configurable.getConfigureName());
+ configurableMap.put(key, configurable);
+ }
+
+ protected void updateConfiguresCache(List<IConfigurable> configureList) {
+ for (IConfigurable iConfigurable : configureList) {
+ updateConfiguresCache(iConfigurable);
+ }
+ }
+
+ protected boolean equals(String key, List<?> newConfigureList) {
+ for (Object o : newConfigureList) {
+ IConfigurable configure = (IConfigurable)o;
+ String tempKey = getConfigureKey(configure.getNameSpace(), configure.getType(), configure.getConfigureName());
+ if (key.equals(tempKey)) {
+ IConfigurable oldConfigure = configurableMap.get(key);
+ if (oldConfigure == null) {
+ continue;
+ }
+ return ConfigurableUtil.compare(oldConfigure, configure);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> queryConfigurableByType(String type) {
+ List<IConfigurable> list = queryConfigurable(type);
+ if (list == null) {
+ return new ArrayList<T>();
+ }
+ List<T> result = new ArrayList<T>();
+ for (IConfigurable configurable : list) {
+ result.add((T)configurable);
+ }
+ return result;
+ }
+
+ @Override
+ public boolean refreshConfigurable(String namespace) {
+ //每次刷新，重新刷新配置文件
+ //if(ComponentCreator.propertiesPath!=null){
+ // ComponentCreator.setProperties(ComponentCreator.propertiesPath);
+ //}
+ this.namespace = namespace;
+ // Map<String, List<IConfigurable>> namespace2ConfigurableMap = new HashMap<>();
+ Map<String, List<IConfigurable>> tempType2ConfigurableMap = new HashMap<>();
+ Map<String, IConfigurable> tempName2ConfigurableMap = new HashMap<>();
+ GetConfigureResult configures = loadConfigurable(namespace);
+ // updateConfiguresCache(configures.getConfigure());
+ if (configures != null && configures.isQuerySuccess() && configures.getConfigurables() != null) {
+ // List<Configure> configureList = filterConfigure(configures.getConfigure());
+ List<IConfigurable> configurables = configures.getConfigurables();
+ List<IConfigurable> configurableList = checkAndUpdateConfigurables(namespace, configurables,
+ tempType2ConfigurableMap, tempName2ConfigurableMap,
+ configures.getConfigurables());
+ // this.namespace2ConfigurableMap = namespace2ConfigurableMap;
+ for (IConfigurable configurable : configurableList) {
+ if (configurable instanceof IAfterConfiguableRefreshListerner) {
+ ((IAfterConfiguableRefreshListerner)configurable).doProcessAfterRefreshConfigurable(this);
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public <T> T queryConfigurable(String configurableType, String name) {
+ return (T)queryConfigurableByIdent(configurableType, name);
+ }
+
+ protected List<IConfigurable> checkAndUpdateConfigurables(String namespace, List<IConfigurable> configurables,
+ Map<String, List<IConfigurable>> tempType2ConfigurableMap,
+ Map<String, IConfigurable> tempName2ConfigurableMap,
+ List configures) {
+ List<IConfigurable> configurableList = new ArrayList<>();
+ for (IConfigurable configurable : configurables) {
+ try {
+ boolean isUpdate = update(configurable, tempName2ConfigurableMap, tempType2ConfigurableMap);
+ if (isUpdate) {
+ configurableList.add(configurable);
+ }
+ } catch (Exception e) {
+ LOG.error("组件初始化异常：" + e.getMessage() + ",name=" + configurable.getConfigureName(), e);
+ }
+ }
+ destroyOldConfigurables(tempName2ConfigurableMap);
+ this.name2ConfigurableMap = tempName2ConfigurableMap;
+ this.type2ConfigurableMap = tempType2ConfigurableMap;
+ return configurableList;
+ }
+
+ private void destroyOldConfigurables(Map<String, IConfigurable> tempName2ConfigurableMap) {
+ Iterator<Map.Entry<String, IConfigurable>> it = this.name2ConfigurableMap.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, IConfigurable> entry = it.next();
+ String key = entry.getKey();
+ IConfigurable value = entry.getValue();
+ if (!tempName2ConfigurableMap.containsKey(key)) {
+ destroyOldConfigurable(value);
+ }
+ }
+
+ }
+
+ private void destroyOldConfigurable(IConfigurable oldConfigurable) {
+ if (AbstractConfigurable.class.isInstance(oldConfigurable)) {
+ ((AbstractConfigurable)oldConfigurable).destroy();
+ }
+ String key = getConfigureKey(oldConfigurable.getNameSpace(), oldConfigurable.getType(),
+ oldConfigurable.getConfigureName());
+ configurableMap.remove(key);
+ }
+
+ protected void initConfigurable(IConfigurable configurable) {
+ if (AbstractConfigurable.class.isInstance(configurable)) {
+ AbstractConfigurable abstractConfigurable = (AbstractConfigurable)configurable;
+ abstractConfigurable.setConfigurableService(this);
+ }
+
+ configurable.init();
+
+ }
+
+ /**
+ * 内部使用
+ */
+ private ScheduledExecutorService scheduledExecutorService;
+
+ @Override
+ public void initConfigurables(final String namespace) {
+ refreshConfigurable(namespace);
+ long polingTime = -1;
+ if (this.properties != null) {
+ String pollingTimeStr = this.properties.getProperty(AbstractComponent.POLLING_TIME);
+ if (StringUtil.isNotEmpty(pollingTimeStr)) {
+ polingTime = Long.valueOf(pollingTimeStr);
+ }
+ }
+ if (polingTime > 0) {
+ scheduledExecutorService = new ScheduledThreadPoolExecutor(3);
+ scheduledExecutorService.scheduleWithFixedDelay(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ refreshConfigurable(namespace);
+ } catch (Exception e) {
+ LOG.error("Load configurables error:" + e.getMessage(), e);
+ }
+ }
+ }, polingTime, polingTime, TimeUnit.SECONDS);
+ }
+ }
+ // @Override
+ // public List<IConfigurable> queryConfigurable(String nameSpace) {
+ // return namespace2ConfigurableMap.get(nameSpace);
+ // }
+
+ @Override
+ public List<IConfigurable> queryConfigurable(String type) {
+ String key = MapKeyUtil.createKey(type);
+ return type2ConfigurableMap.get(key);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String type, String name) {
+ String key = MapKeyUtil.createKey(type, name);
+ return name2ConfigurableMap.get(key);
+ }
+
+ /**
+ * 根据namespace加载配置信息
+ *
+ * @param namespace
+ * @return
+ * @throws Exception
+ */
+ protected abstract GetConfigureResult loadConfigurable(String namespace);
+
+ @Override
+ public void update(IConfigurable configurable) {
+ // update(configurable,name2ConfigurableMap,type2ConfigurableMap);
+ updateConfigurable(configurable);
+ }
+
+ protected abstract void updateConfigurable(IConfigurable configurable);
+
+ protected abstract void insertConfigurable(IConfigurable configurable);
+
+ protected boolean update(IConfigurable configurable, Map<String, IConfigurable> name2ConfigurableMap,
+ Map<String, List<IConfigurable>> type2ConfigurableMap) {
+ if (configurable == null) {
+ return false;
+ }
+
+ boolean isUpdate = false;
+ List<IConfigurable> configurableList = new ArrayList<>();
+ configurableList.add(configurable);
+
+ String nameKey = MapKeyUtil.createKey(configurable.getType(), configurable.getConfigureName());
+ if (this.name2ConfigurableMap.containsKey(nameKey)) {
+ String configureKey = getConfigureKey(namespace, configurable.getType(), configurable.getConfigureName());
+ IConfigurable oldConfigurable = this.name2ConfigurableMap.get(nameKey);
+ if (equals(configureKey, configurableList)) {
+ configurable = oldConfigurable;
+ // name2ConfigurableMap.put(nameKey, name2ConfigurableMap.get(nameKey));
+ } else {
+ destroyOldConfigurable(oldConfigurable);
+ initConfigurable(configurable);
+ isUpdate = true;
+ }
+ } else {
+ initConfigurable(configurable);
+ isUpdate = true;
+ }
+ updateConfiguresCache(configurable);
+ name2ConfigurableMap.put(nameKey, configurable);
+ String typeKey = MapKeyUtil.createKey(configurable.getType());
+ // put2Map(namespace2ConfigurableMap, namespace, configurable);
+ put2Map(type2ConfigurableMap, typeKey, configurable);
+ return isUpdate;
+ }
+
+ @Override
+ public void insert(IConfigurable configurable) {
+ // update(configurable,name2ConfigurableMap,type2ConfigurableMap);
+ insertConfigurable(configurable);
+ }
+
+ /**
+ * 给一个扣，可以跨命名空间查询数据
+ *
+ * @param namespaces
+ * @return
+ */
+ public List<IConfigurable> queryConfiguableByNamespace(String... namespaces) {
+ List<IConfigurable> configurables = new ArrayList<>();
+ if (namespaces == null || namespaces.length == 0) {
+ return configurables;
+ }
+ for (String namespace : namespaces) {
+ GetConfigureResult result = loadConfigurable(namespace);
+ if (result.querySuccess) {
+ if (result.configurables != null && result.configurables.size() > 0) {
+ configurables.addAll(result.configurables);
+ }
+ } else {
+ throw new RuntimeException("Load configurable error, the namespace is " + namespace);
+ }
+ }
+ return configurables;
+
+ }
+
+ /**
+ * 往一个value是list的map中添加数据，如果list是空创建，否则直接插入
+ *
+ * @param map
+ * @param key
+ * @param configurable
+ */
+ protected void put2Map(Map<String, List<IConfigurable>> map, String key, IConfigurable configurable) {
+ List<IConfigurable> list = map.computeIfAbsent(key, k -> new ArrayList<IConfigurable>());
+ list.add(configurable);
+ }
+
+ @Override
+ public Collection<IConfigurable> findAll() {
+ return name2ConfigurableMap.values();
+ }
+
+ /**
+ * 把configurable转换成configure
+ *
+ * @param configurable
+ * @return
+ */
+ protected Configure createConfigure(IConfigurable configurable) {
+ Configure configure = new Configure();
+ configure.setType(configurable.getType());
+ configure.setName(configurable.getConfigureName());
+ configure.setNameSpace(configurable.getNameSpace());
+ String jsonString = configurable.toJson();
+ if (!StringUtil.isEmpty(jsonString)) {
+ JSONObject jsonObject = JSONObject.parseObject(jsonString);
+ jsonObject.put(CLASS_NAME, configurable.getClass().getName());
+ configure.setJsonValue(jsonObject.toJSONString());
+ }
+ // configure.createIdentification();
+ return configure;
+ }
+
+ @Override
+ public <T> Map<String, T> queryConfigurableMapByType(String type) {
+ List<IConfigurable> configurables = queryConfigurable(type);
+ if (configurables == null) {
+ return new HashMap<String, T>();
+ }
+ Map<String, T> result = new HashMap<String, T>();
+ for (IConfigurable configurable : configurables) {
+ result.put(configurable.getConfigureName(), (T)configurable);
+ }
+ return result;
+ }
+
+ /**
+ * 把configure转换成configurable
+ *
+ * @param configures
+ * @return
+ */
+ protected List<IConfigurable> convert(List<Configure> configures) {
+ if (configures == null) {
+ return new ArrayList<IConfigurable>();
+ }
+ List<IConfigurable> configurables = new ArrayList<IConfigurable>();
+ for (Configure configure : configures) {
+ IConfigurable configurable = convert(configure);
+ if (configurable != null) {
+ configurables.add(configurable);
+ }
+
+ }
+ return configurables;
+ }
+
+ protected IConfigurable createConfigurableFromJson(String namespace, String type, String name, String jsonValue) {
+ if (StringUtil.isEmpty(jsonValue)) {
+ return null;
+ }
+ JSONObject jsonObject = JSONObject.parseObject(jsonValue);
+ String className = jsonObject.getString(CLASS_NAME);
+ IConfigurable configurable = createConfigurable(className);
+ if (configurable == null) {
+ return null;
+ }
+ configurable.setConfigureName(name);
+ configurable.setNameSpace(namespace);
+ configurable.setType(type);
+ if (AbstractConfigurable.class.isInstance(configurable)) {
+ AbstractConfigurable abstractConfigurable = (AbstractConfigurable)configurable;
+ abstractConfigurable.setConfigurableService(this);
+ }
+ configurable.toObject(jsonValue);
+ return configurable;
+ }
+
+ /**
+ * 提供一个入口，可以让外部用户改变configure对应的configurable的值
+ *
+ * @param configure
+ * @return
+ */
+ protected IConfigurable convert(Configure configure) {
+
+ return convertConfigurable(configure);
+ }
+
+ protected IConfigurable convertConfigurable(Configure configure) {
+ String className = null;
+ try {
+ String jsonString = configure.getJsonValue();
+ IConfigurable configurable =
+ createConfigurableFromJson(configure.getNameSpace(), configure.getType(), configure.getName(),
+ jsonString);
+ if (configurable instanceof Entity) {
+ // add by wangtl 20171110 Configurable接口第三方包也在用，故不能Configurable里加接口，只能加到抽象类里，这里强转下
+ Entity abs = (Entity)configurable;
+ abs.setId(configure.getId());
+ abs.setGmtCreate(configure.getGmtCreate());
+ abs.setGmtModified(configure.getGmtModified());
+ /*
+ * abs.setTempKey((configurable.getNameSpace() + configurable.getType() +
+ * configurable.getConfigureName() + jsonString).hashCode());
+ */
+ }
+ convertPost(configurable);
+ return configurable;
+ } catch (Exception e) {
+ LOG.error("转换异常：" + configure.toString(), e);
+ return null;
+ }
+ }
+
+ /**
+ * 如果需要改变configurable的属性，可以再这里设置
+ *
+ * @param configurable
+ */
+ @SuppressWarnings("rawtypes")
+ protected void convertPost(IConfigurable configurable) {
+ if (this.properties == null) {
+ return;
+ }
+ String identification =
+ MapKeyUtil.createKey(configurable.getNameSpace(), configurable.getType(), configurable.getConfigureName());
+ String propertyValue = this.properties.getProperty(identification);
+ if (StringUtil.isEmpty(propertyValue)) {
+ return;
+ }
+ String[] fieldName2Values = propertyValue.split(",");
+ if (fieldName2Values.length == 0) {
+ return;
+ }
+ for (String fieldName2Value : fieldName2Values) {
+ try {
+ String[] fieldName2ValueArray = fieldName2Value.split(":");
+ if (fieldName2ValueArray.length != 2) {
+ continue;
+ }
+ String fieldName = fieldName2ValueArray[0];
+ String value = fieldName2ValueArray[1];
+ Class clazz = ReflectUtil.getBeanFieldType(configurable.getClass(), fieldName);
+ DataType dataType = DataTypeUtil.createDataType(clazz, null);
+ if (dataType == null) {
+ continue;
+ }
+ Object fieldValue = dataType.getData(value);
+ ReflectUtil.setBeanFieldValue(configurable, fieldName, fieldValue);
+
+ } catch (Exception e) {
+ LOG.error("convert post error " + fieldName2Value, e);
+ continue;
+ }
+
+ }
+ }
+
+ /**
+ * 创建configurable对象
+ *
+ * @param className class name
+ * @return
+ */
+ @SuppressWarnings("rawtypes")
+ protected IConfigurable createConfigurable(String className) {
+ return ReflectUtil.forInstance(className);
+ }
+
+ public class GetConfigureResult {
+
+ private boolean querySuccess;
+ private List<IConfigurable> configurables;
+
+ public boolean isQuerySuccess() {
+ return querySuccess;
+ }
+
+ public void setQuerySuccess(boolean querySuccess) {
+ this.querySuccess = querySuccess;
+ }
+
+ public List<IConfigurable> getConfigurables() {
+ return configurables;
+ }
+
+ public void setConfigurables(List<IConfigurable> configurables) {
+ this.configurables = configurables;
+ }
+ }
+
+ @Override
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractSupportParentConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractSupportParentConfigureService.java
new file mode 100644
index 0000000..390a119
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/AbstractSupportParentConfigureService.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableListerner;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * namespace 分层，支持顶级命名空间，顶级命名空间的对象，所有命名空间都可见。顶级命名空间是固定值IConfigurableService.PARENT_CHANNEL_NAME_SPACE
+ */
+public abstract class AbstractSupportParentConfigureService extends AbstractConfigurableService
+ implements IConfigurableService {
+
+ private static final Log LOG = LogFactory.getLog(AbstractSupportParentConfigureService.class);
+ protected IConfigurableService configureService = null;
+ protected IConfigurableService parentConfigureService = null;
+ //protected IConfigurableService shareConfigureService = null;
+ protected Properties properties;
+
+ public AbstractSupportParentConfigureService() {
+ super(null);
+ }
+
+ public void initMethod(Properties property) {
+ this.properties = property;
+ initBeforeInitConfigurable(property);
+ }
+
+ protected abstract void initBeforeInitConfigurable(Properties property);
+
+
+
+
+ @Override
+ public void initConfigurables(String namespace) {
+
+ if (!IConfigurableService.PARENT_CHANNEL_NAME_SPACE.equals(namespace)) {
+ parentConfigureService.initConfigurables(IConfigurableService.PARENT_CHANNEL_NAME_SPACE);
+ } else {
+ parentConfigureService = null;
+ }
+ configureService.initConfigurables(namespace);
+ }
+
+ @Override
+ public boolean refreshConfigurable(String namespace) {
+
+ if (!IConfigurableService.PARENT_CHANNEL_NAME_SPACE.equals(namespace)) {
+ parentConfigureService.refreshConfigurable(IConfigurableService.PARENT_CHANNEL_NAME_SPACE);
+ // initShareConfigurableService(namespace);
+ }
+ configureService.refreshConfigurable(namespace);
+ return true;
+ }
+
+
+
+ @Override
+ public List<IConfigurable> queryConfigurable(String type) {
+ List<IConfigurable> result = configureService.queryConfigurable(type);
+ if (result == null) {
+ result = new ArrayList<>();
+ }
+ //if (shareConfigureService != null) {
+ // List<IConfigurable> share = shareConfigureService.queryConfigurable(type);
+ // if (share != null) {
+ // result.addAll(share);
+ // }
+ //}
+ if (parentConfigureService == null) {
+ return result;
+ }
+ List<IConfigurable> parent = parentConfigureService.queryConfigurable(type);
+ if (parent != null) {
+ result.addAll(parent);
+ }
+ return result;
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String type, String name) {
+ IConfigurable configurable = configureService.queryConfigurableByIdent(type, name);
+ if (configurable != null) {
+ return configurable;
+ }
+ if (parentConfigureService == null) {
+ return null;
+ }
+ //if (shareConfigureService != null) {
+ // configurable = shareConfigureService.queryConfigurableByIdent(type, name);
+ //}
+ if (configurable != null) {
+ return configurable;
+ }
+ return parentConfigureService.queryConfigurableByIdent(type, name);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String identification) {
+ IConfigurable configurable = configureService.queryConfigurableByIdent(identification);
+ if (configurable != null) {
+ return configurable;
+ }
+ if (parentConfigureService == null) {
+ return null;
+ }
+ //if (shareConfigureService != null) {
+ // configurable = shareConfigureService.queryConfigurableByIdent(identification);
+ //}
+ if (configurable != null) {
+ return configurable;
+ }
+ return parentConfigureService.queryConfigurableByIdent(identification);
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configurable) {
+ if (parentConfigureService != null && configurable.getNameSpace()
+ .equals(IConfigurableService.PARENT_CHANNEL_NAME_SPACE)) {
+ parentConfigureService.insert(configurable);
+ } else {
+ configureService.insert(configurable);
+ }
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configurable) {
+ if (parentConfigureService != null && configurable.getNameSpace()
+ .equals(IConfigurableService.PARENT_CHANNEL_NAME_SPACE)) {
+ parentConfigureService.update(configurable);
+ } else {
+ configureService.update(configurable);
+ }
+ }
+
+ @Override
+ public <T> T queryConfigurable(String configurableType, String name) {
+ return (T) queryConfigurableByIdent(configurableType, name);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ return null;
+ }
+
+ //protected void initShareConfigurableService(String namespace) {
+ // if (parentConfigureService == null) {
+ // return;
+ // }
+ // shareConfigureService = new AbstractReadOnlyConfigurableService() {
+ //
+ // @Override
+ // public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ // refreshConfigurable(namespace);
+ // return queryConfigurableByType(type);
+ // }
+ //
+ // @Override
+ // protected List<IConfigurable> loadConfigurables(String namespace) {
+ // List<IConfigurable> parent = parentConfigureService.queryConfigurable(ShareConfiguable.TYPE);
+ // List<IConfigurable> shareConfigurables = new ArrayList<>();
+ // if (parent == null) {
+ // return shareConfigurables;
+ // }
+ // for (IConfigurable configurable : parent) {
+ // ShareConfiguable shareConfiguable = (ShareConfiguable) configurable;
+ // if (shareConfiguable.getShareAll() || shareConfiguable.getShareNameSpaces().contains(namespace)) {
+ // String sharedNameSpace = shareConfiguable.getSharedNameSpace();
+ // String sharedType = shareConfiguable.getSharedType();
+ // String sharedName = shareConfiguable.getSharedName();
+ // List<IConfigurable> sharedConfigrables =
+ // createAndQueryConfigurable(sharedNameSpace, sharedType, sharedName);
+ // if (sharedConfigrables != null) {
+ // shareConfigurables.addAll(sharedConfigrables);
+ // }
+ // }
+ // }
+ // return shareConfigurables;
+ // }
+ //
+ //
+ // };
+ // shareConfigureService.refreshConfigurable(namespace);
+ //
+ //}
+
+ protected List<IConfigurable> createAndQueryConfigurable(String sharedNameSpace, String sharedType,
+ String sharedName) {
+ IConfigurableService innerSharedConfigurableService =
+ ConfigurableServiceFactory.createConfigurableService(properties);
+ innerSharedConfigurableService.refreshConfigurable(sharedNameSpace);
+ if (StringUtil.isNotEmpty(sharedName)) {
+ List<IConfigurable> configurables = new ArrayList<>();
+ IConfigurable configurable = innerSharedConfigurableService.queryConfigurableByIdent(sharedType, sharedName);
+ configurables.add(configurable);
+ return configurables;
+ } else {
+ return innerSharedConfigurableService.queryConfigurable(sharedType);
+ }
+
+ }
+
+
+ @Override
+ public Collection<IConfigurable> findAll() {
+ List<IConfigurable> configurables=new ArrayList<>();
+ if (parentConfigureService != null ) {
+ Collection<IConfigurable> tmp=parentConfigureService.findAll();
+ if(tmp!=null||tmp.size()>0){
+ configurables.addAll(tmp);
+ }
+ }
+ Collection<IConfigurable> tmp=configureService.findAll();
+ if(tmp!=null||tmp.size()>0){
+ configurables.addAll(tmp);
+ }
+ return configurables;
+ }
+
+ public IConfigurableService getConfigureService() {
+ return configureService;
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ List<T> configurables=new ArrayList<>();
+ if (parentConfigureService != null ) {
+ Collection<T> tmp=parentConfigureService.loadConfigurableFromStorage(type);
+ if(tmp!=null||tmp.size()>0){
+ configurables.addAll(tmp);
+ }
+ }
+ Collection<T> tmp=configureService.loadConfigurableFromStorage(type);
+ if(tmp!=null||tmp.size()>0){
+ configurables.addAll(tmp);
+ }
+ return configurables;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServcieType.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServcieType.java
new file mode 100644
index 0000000..f2b72ab
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServcieType.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+
+@Deprecated
+public class ConfigurableServcieType {
+
+ public static final String DEFAULT_SERVICE_NAME = IConfigurableService.DEFAULT_SERVICE_NAME;
+ public static final String MEMORY_SERVICE_NAME = IConfigurableService.MEMORY_SERVICE_NAME;
+ public static final String FILE_SERVICE_NAME = IConfigurableService.FILE_SERVICE_NAME;
+ public static final String HTTP_SERVICE_NAME = IConfigurableService.HTTP_SERVICE_NAME;
+
+
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServiceFactory.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServiceFactory.java
new file mode 100644
index 0000000..307dcae
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/ConfigurableServiceFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service;
+
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.serviceloader.ServiceLoaderComponent;
+
+public class ConfigurableServiceFactory {
+ private static final ServiceLoaderComponent<IConfigurableService> configurableServiceLoaderComponent = ServiceLoaderComponent.getInstance(IConfigurableService.class);
+ private static final Log LOG = LogFactory.getLog(ConfigurableServiceFactory.class);
+
+ public static IConfigurableService createConfigurableService(Properties properties) {
+ try {
+ Properties properties1 = new Properties();
+ properties1.putAll(properties);
+ String type = properties1.getProperty(ConfigureFileKey.CONNECT_TYPE);
+ if (StringUtil.isEmpty(type)) {
+ type = IConfigurableService.DEFAULT_SERVICE_NAME;
+ }
+ IConfigurableService configurableService = getConfigurableServiceType(type);
+ if (configurableService instanceof AbstractSupportParentConfigureService) {
+ ((AbstractSupportParentConfigureService)configurableService).initMethod(properties1);
+ }
+ return configurableService;
+ } catch (Exception e) {
+ LOG.error("create ConfigurableService error", e);
+ return null;
+ }
+
+ }
+
+ public static IConfigurableService getConfigurableServiceType(String type) {
+ IConfigurableService configurableService = configurableServiceLoaderComponent.getService().loadService(type);
+ return ReflectUtil.forInstance(configurableService.getClass().getName());
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileConfigureService.java
new file mode 100644
index 0000000..d251bda
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileConfigureService.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service.impl;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configuable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.configuable.model.Configure;
+import org.apache.rocketmq.streams.common.model.Entity;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+public class FileConfigureService extends AbstractConfigurableService {
+
+ public static final String FILE_PATH_NAME = IConfigurableService.FILE_PATH_NAME;
+ // 配置文件的路径
+ private static final Log LOG = LogFactory.getLog(FileConfigureService.class);
+ private static final String DEFAULT_FILE_NAME = "dipper_configure.cs"; // 默认文件名
+ private static final String SIGN = "&&&&"; // 字段分割附号
+ public String fileName;
+
+ public FileConfigureService(Properties properties) {
+ super(properties);
+ initService(properties.getProperty(FILE_PATH_NAME));
+ }
+
+ protected void initService(String fileAndPath) {
+ if (StringUtil.isEmpty(fileAndPath)) {
+ String path = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
+ if (path.endsWith(".jar")) {
+ int index = path.lastIndexOf(File.separator);
+ path = path.substring(0, index);
+ }
+ fileName = FileUtil.concatFilePath(path, DEFAULT_FILE_NAME);
+ } else {
+ fileName = fileAndPath;
+ }
+ LOG.info("load file from path = " + fileName);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ GetConfigureResult result = new GetConfigureResult();
+ try {
+ List<Configure> configures = selectOpening(namespace);
+ List<IConfigurable> configurables = convert(configures);
+ LOG.info("load configure namespace=" + namespace + " count=" + configures.size());
+ result.setConfigurables(configurables);
+ result.setQuerySuccess(true);// 该字段标示查询是否成功，若不成功则不会更新配置
+ } catch (Exception e) {
+ result.setQuerySuccess(false);
+ e.printStackTrace();
+ LOG.error("load configurable error ", e);
+ }
+ return result;
+ }
+
+ protected List<Configure> selectOpening(String namespace) {
+ List<String> list = loadFileLine(fileName);
+ List<Configure> configures = convert2Configure(list);
+ return filter(configures, namespace);
+ }
+
+ protected List<Configure> filter(List<Configure> configures, String namespace) {
+ if (configures == null) {
+ return new ArrayList<>();
+ }
+ if (StringUtil.isEmpty(namespace)) {
+ throw new RuntimeException("namespace can not empty ");
+ }
+ List<Configure> filterConfigures = new ArrayList<>();
+ for (Configure configure : configures) {
+ if (!namespace.equals(configure.getNameSpace())) {
+ continue;
+ }
+ filterConfigures.add(configure);
+ }
+ return filterConfigures;
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configure) {
+ if (configure == null) {
+ LOG.warn("insert configure is null");
+ return;
+ }
+ String row = configure2String(configure);
+
+ List<String> rows = loadFileLine(fileName);
+ if (rows == null) {
+ rows = new ArrayList<>();
+ }
+ List<Configure> configures = convert2Configure(rows);
+ String newKey =
+ MapKeyUtil.createKey(configure.getNameSpace(), configure.getType(), configure.getConfigureName());
+ boolean isReplace = false;
+ for (int i = 0; i < configures.size(); i++) {
+ Configure c = configures.get(i);
+ String old = MapKeyUtil.createKey(c.getNameSpace(), c.getType(), c.getName());
+ if (old.equals(newKey)) {
+ rows.set(i, configure2String(configure));
+ isReplace = true;
+ break;
+ }
+ }
+ if (!isReplace) {
+ rows.add(configure2String(configure));
+ }
+ writeFile(fileName, rows);
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configure) {
+ if (configure == null) {
+ LOG.warn("insert configure is null");
+ return;
+ }
+
+ List<String> rows = FileUtil.loadFileLine(fileName);
+ if (rows == null) {
+ rows = new ArrayList<>();
+ }
+ for (int i = 0; i < rows.size(); i++) {
+ String row = rows.get(i);
+ Configure oldConfigure = convert(row);
+ if (configure.getNameSpace().equals(oldConfigure.getNameSpace()) && configure.getType()
+ .equals(oldConfigure.getType()) && configure.getConfigureName().equals(oldConfigure.getName())) {
+ rows.set(i, configure2String(configure));
+ }
+ }
+ writeFile(fileName, rows);
+
+ }
+
+ protected Configure convert(String row) {
+ String[] values = row.split(SIGN);
+ String namespace = getColumnValue(values, 0, "namespace");
+ String type = getColumnValue(values, 1, "type");
+ String name = getColumnValue(values, 2, "name");
+ String jsonValue = getColumnValue(values, 3, "json_value");
+ String createDate = getColumnValue(values, 4, "gmt_create");
+ String modifiedDate = getColumnValue(values, 5, "gmt_modified");
+ String id = getColumnValue(values, 6, "id");
+ Configure configure = new Configure();
+ configure.setNameSpace(namespace);
+ configure.setType(type);
+ configure.setName(name);
+ configure.setJsonValue(jsonValue);
+ configure.setGmtCreate(DateUtil.parse(createDate));
+ configure.setGmtCreate(DateUtil.parse(modifiedDate));
+ configure.setId((id == null ? null : Long.valueOf(id)));
+
+ return configure;
+ }
+
+ protected List<Configure> convert2Configure(List<String> rows) {
+ List<Configure> configures = new ArrayList<Configure>();
+ for (String row : rows) {
+ configures.add(convert(row));
+ }
+ return configures;
+ }
+
+ protected String getColumnValue(String[] values, int i, String namespace) {
+ if (values == null || values.length == 0) {
+ return null;
+ }
+ if (values.length <= i) {
+ return null;
+ }
+ if ("null".equals(values[i])) {
+ return null;
+ }
+ return values[i];
+ }
+
+ /**
+ * 解密文件，并加载到内存
+ *
+ * @param fileName
+ * @return
+ */
+ protected List<String> loadFileLine(String fileName) {
+ List<String> rows = FileUtil.loadFileLine(fileName);
+ if (rows == null) {
+ rows = new ArrayList<>();
+ }
+ return doDecRowList(rows);
+ }
+
+ protected void writeFile(String fileName, List<String> rows) {
+ List<String> rowList = doEncryptRowList(rows);
+ FileUtil.write(fileName, rowList);
+ }
+
+ private List<String> doEncryptRowList(List<String> rows) {
+ return rows;
+ }
+
+ private List<String> doDecRowList(List<String> rows) {
+ return rows;
+ }
+
+ protected String configure2String(IConfigurable configure) {
+ Entity entity = null;
+ if (configure instanceof Entity) {
+ entity = (Entity)configure;
+ } else {
+ entity = new Entity();
+ }
+ String row = MapKeyUtil.createKeyBySign(SIGN, configure.getNameSpace(), configure.getType(),
+ configure.getConfigureName(), configure.toJson(), DateUtil.format(entity.getGmtCreate()),
+ DateUtil.format(entity.getGmtModified()), entity.getId() + "");
+ return row;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ refreshConfigurable(getNamespace());
+ return queryConfigurableByType(type);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileSupportParentConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileSupportParentConfigureService.java
new file mode 100644
index 0000000..bf8d151
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/FileSupportParentConfigureService.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service.impl;
+
+import org.apache.rocketmq.streams.configuable.service.AbstractSupportParentConfigureService;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configuable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import com.google.auto.service.AutoService;
+
+import java.util.Properties;
+
+@AutoService(IConfigurableService.class)
+@ServiceName(ConfigurableServcieType.FILE_SERVICE_NAME)
+public class FileSupportParentConfigureService extends AbstractSupportParentConfigureService {
+
+ @Override
+ protected void initBeforeInitConfigurable(Properties property) {
+ this.configureService = new FileConfigureService(properties);
+ this.parentConfigureService = new FileConfigureService(properties);
+ }
+
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemoryConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemoryConfigureService.java
new file mode 100644
index 0000000..e846e44
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemoryConfigureService.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.configuable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+
+public class MemoryConfigureService extends AbstractConfigurableService {
+
+ private static Map<String, List<IConfigurable>> namespace2Configure = new HashMap<>();
+
+ public MemoryConfigureService(Properties properties) {
+ super(properties);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ GetConfigureResult result = new GetConfigureResult();
+ result.setQuerySuccess(true);
+ List<IConfigurable> configurableList = new ArrayList<>();
+ List<IConfigurable> configurables = namespace2Configure.get(namespace);
+ if (configurables == null) {
+ configurableList = null;
+ } else {
+ List<IConfigurable> tmps = new ArrayList<>();
+ tmps.addAll(configurables);
+ for (IConfigurable configurable : tmps) {
+ IConfigurable tmp = ReflectUtil.forInstance(configurable.getClass());
+ tmp.toObject(configurable.toJson());
+ tmp.setNameSpace(configurable.getNameSpace());
+ tmp.setConfigureName(configurable.getConfigureName());
+ configurableList.add(tmp);
+ }
+ }
+ result.setConfigurables(configurableList);
+ return result;
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configurable) {
+ if (configurable == null) {
+ return;
+ }
+
+ String namespace = configurable.getNameSpace();
+ List<IConfigurable> list = namespace2Configure.get(namespace);
+ if (list == null) {
+ synchronized (this) {
+ list = namespace2Configure.get(namespace);
+ if (list == null) {
+ list = new ArrayList<>();
+ namespace2Configure.put(namespace, list);
+ }
+ }
+ }
+ int removeIndex = -1;
+ for (int i = 0; i < list.size(); i++) {
+ IConfigurable config = list.get(i);
+ if (config.getType().equals(configurable.getType()) && config.getConfigureName()
+ .equals(configurable.getConfigureName())) {
+ removeIndex = i;
+ }
+ }
+ if (AbstractConfigurable.class.isInstance(configurable)) {
+ ((AbstractConfigurable)configurable).setConfigurableService(this);
+ }
+ if (removeIndex != -1) {
+ list.remove(removeIndex);
+ }
+ list.add(configurable);
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configure) {
+ List<IConfigurable> list = namespace2Configure.get(configure.getNameSpace());
+ if (list == null || list.size() == 0) {
+ throw new RuntimeException(
+ "not have exist configure " + configure.getNameSpace() + "," + configure.getType() + ","
+ + configure.getConfigureName());
+ }
+ for (int i = 0; i < list.size(); i++) {
+ IConfigurable config = list.get(i);
+ if (config.getType().equals(configure.getType()) && config.getConfigureName()
+ .equals(configure.getConfigureName())) {
+ list.set(i, configure);
+ return;
+ }
+ }
+ throw new RuntimeException(
+ "not have exist configure " + configure.getNameSpace() + "," + configure.getType() + ","
+ + configure.getConfigureName());
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ refreshConfigurable(getNamespace());
+ return queryConfigurableByType(type);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemorySupportParentConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemorySupportParentConfigureService.java
new file mode 100644
index 0000000..528d087
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configuable/service/impl/MemorySupportParentConfigureService.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.service.impl;
+
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.configuable.service.AbstractSupportParentConfigureService;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configuable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import com.google.auto.service.AutoService;
+
+
+@AutoService(IConfigurableService.class)
+@ServiceName(ConfigurableServcieType.MEMORY_SERVICE_NAME)
+public class MemorySupportParentConfigureService extends AbstractSupportParentConfigureService {
+
+ @Override
+ protected void initBeforeInitConfigurable(Properties property) {
+ // this.rootConfigureService = new MemoryConfigureService(property);
+ this.parentConfigureService = new MemoryConfigureService(properties);
+ this.configureService = new MemoryConfigureService(properties);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/ConfigurableComponent.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/ConfigurableComponent.java
new file mode 100644
index 0000000..91b3171
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/ConfigurableComponent.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.component.ConfigureDescriptor;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configurable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServiceFactory;
+import org.apache.rocketmq.streams.common.utils.ConfigurableUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * 对Configurable对象，做统一的管理，统一查询，插入和更新。 insert/update 把configuabel对象写入存储，支持文件存储（file），内存存储（memory）和db存储（DB）。可以在配置通过这个ConfigureFileKey.CONNECT_TYPE key 配置 query 是基于内存的查询，对象定时load到内存，可以在属性文件通过这个ConfigureFileKey.POLLING_TIME key配置加载周期，单位是秒 新对象加载后生效，已经存在的对象只有updateFlag发生变化才会被替换
+ */
+public class ConfigurableComponent extends AbstractComponent<IConfigurableService>
+ implements IConfigurableService {
+
+ private static final Log LOG = LogFactory.getLog(ConfigurableComponent.class);
+
+ protected volatile IConfigurableService configureService = null;
+
+ protected transient String namespace;
+
+ public ConfigurableComponent() {
+ initConfigurableServiceDescriptor();
+ addConfigureDescriptor(
+ new ConfigureDescriptor(CONNECT_TYPE, false, ConfigurableServcieType.DEFAULT_SERVICE_NAME));
+ }
+
+ public static ConfigurableComponent getInstance(String namespace) {
+ return ComponentCreator.getComponent(namespace, ConfigurableComponent.class);
+ }
+
+ @Override
+ protected boolean initProperties(Properties properties) {
+ try {
+ if (configureService != null) {
+ return true;
+ }
+ this.configureService = ConfigurableServiceFactory.createConfigurableService(properties);
+ return true;
+ } catch (Exception e) {
+ LOG.error("ConfigurableComponent create error,properties= " + properties, e);
+ return false;
+ }
+
+ }
+
+ @Override
+ public boolean startComponent(String namespace) {
+ try {
+ this.namespace = namespace;
+ configureService.initConfigurables(namespace);
+ return true;
+ } catch (Exception e) {
+ LOG.error("ConfigurableComponent init error, namespace is " + namespace, e);
+ return false;
+ }
+
+ }
+
+ /**
+ * 启动测试模式，用内存数据库存储和加载configurable数据
+ */
+ public static void begineTestMode() {
+ System.setProperty(ConfigurableComponent.CONNECT_TYPE, ConfigurableServcieType.MEMORY_SERVICE_NAME);
+ }
+
+ /**
+ * 关闭测试模式，用配置文件中配置的属性加载configuable数据
+ */
+ public static void endTestMode() {
+ System.clearProperty(ConfigurableComponent.CONNECT_TYPE);
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ @Override
+ public IConfigurableService getService() {
+ return configureService;
+ }
+
+ @Override
+ public void initConfigurables(String namespace) {
+ configureService.initConfigurables(namespace);
+ }
+
+ @Override
+ public boolean refreshConfigurable(String namespace) {
+ return configureService.refreshConfigurable(namespace);
+ }
+
+ public void mockConfigurable(String namespace) {
+ refreshConfigurable(namespace);
+
+ }
+
+ @Override
+ public List<IConfigurable> queryConfigurable(String type) {
+ return configureService.queryConfigurable(type);
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> queryConfigurableByType(String type) {
+ return configureService.queryConfigurableByType(type);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String type, String name) {
+ return configureService.queryConfigurableByIdent(type, name);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String identification) {
+ return configureService.queryConfigurableByIdent(identification);
+ }
+
+ @Override
+ public void insert(IConfigurable configurable) {
+ configureService.insert(configurable);
+ ConfigurableUtil.refreshMock(configurable);
+ }
+
+ @Override
+ public void update(IConfigurable configurable) {
+ configureService.update(configurable);
+ }
+
+ @Override
+ public <T> Map<String, T> queryConfigurableMapByType(String type) {
+ return configureService.queryConfigurableMapByType(type);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T queryConfigurable(String configurableType, String name) {
+ return (T)queryConfigurableByIdent(configurableType, name);
+ }
+
+ //protected void insertConfigurable(JSONObject message, IConfigurable configurable) {
+ // ConfigurableUtil.insertConfigurable(message, configurable, this.configureService);
+ //}
+
+ @Override
+ public String getNamespace() {
+ if (AbstractConfigurableService.class.isInstance(configureService)) {
+ return ((AbstractConfigurableService)configureService).getNamespace();
+ }
+ return namespace;
+ }
+
+ @Override
+ public Collection<IConfigurable> findAll() {
+ return configureService.findAll();
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ return configureService.loadConfigurableFromStorage(type);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/model/Configure.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/model/Configure.java
new file mode 100644
index 0000000..bb4531f
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/model/Configure.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.model;
+
+import org.apache.rocketmq.streams.common.model.Entity;
+
+/**
+ * configuable如果存储在db，这个是db表的映射对象
+ */
+public class Configure extends Entity {
+
+ private static final long serialVersionUID = 5668017348345235669L;
+
+ private String nameSpace;
+ private String type;
+ private String name;
+ // private String identification;
+ private String jsonValue;
+ private String modifyTime;
+ private String remark;
+ private int openRange;
+
+ public static String createTableSQL(String tableName) {
+ return "/******************************************/\n"
+ + "/* TableName = dipper_configure */\n"
+ + "/******************************************/\n"
+ + "CREATE TABLE IF NOT EXISTS `" + tableName + "` (\n"
+ + " `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',\n"
+ + " `gmt_create` datetime NOT NULL COMMENT '创建时间',\n"
+ + " `gmt_modified` datetime NOT NULL COMMENT '修改时间',\n"
+ + " `namespace` varchar(32) NOT NULL COMMENT '项目标识',\n"
+ + " `type` varchar(32) NOT NULL COMMENT '配置类型',\n"
+ + " `name` varchar(128) NOT NULL COMMENT '配置名称',\n"
+ + " `json_value` text NOT NULL COMMENT '配置内容',\n"
+ + " `status` tinyint(3) unsigned NOT NULL DEFAULT '0' COMMENT '1:正在使用 0:已失效',\n"
+ + " PRIMARY KEY (`id`),\n"
+ + " UNIQUE KEY `uk_namespace_type_name` (`namespace`,`type`,`name`),\n"
+ + " KEY `idx_namespace` (`namespace`)\n"
+ + ") ENGINE=InnoDB AUTO_INCREMENT=1814834 DEFAULT CHARSET=utf8 COMMENT='统一接入配置项'\n"
+ + ";";
+ }
+
+ public String getNameSpace() {
+ return nameSpace;
+ }
+
+ public void setNameSpace(String nameSpace) {
+ this.nameSpace = nameSpace;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ // public String getIdentification() {
+ // return identification;
+ // }
+
+ // public void createIdentification() {
+ // this.identification = MapKeyUtil.createKey(nameSpace, type, name);
+ // }
+
+ public String getJsonValue() {
+ return jsonValue;
+ }
+
+ public void setJsonValue(String jsonValue) {
+ this.jsonValue = jsonValue;
+ }
+
+ public String getModifyTime() {
+ return modifyTime;
+ }
+
+ public void setModifyTime(String modifyTime) {
+ this.modifyTime = modifyTime;
+ }
+
+ public String getRemark() {
+ return remark;
+ }
+
+ public void setRemark(String remark) {
+ this.remark = remark;
+ }
+
+ public int getOpenRange() {
+ return openRange;
+ }
+
+ public void setOpenRange(int openRange) {
+ this.openRange = openRange;
+ }
+
+ @Override
+ public String toString() {
+ return "Configure{" + "nameSpace='" + nameSpace + '\'' + ", type='" + type + '\'' + ", name='" + name + '\''
+ + ", jsonValue='" + jsonValue + '\'' + ", modifyTime='" + modifyTime + '\'' + ", remark='" + remark + '\''
+ + ", openRange=" + openRange + '}';
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractConfigurableService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractConfigurableService.java
new file mode 100644
index 0000000..85ef626
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractConfigurableService.java
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.datatype.DataType;
+import org.apache.rocketmq.streams.common.model.Entity;
+import org.apache.rocketmq.streams.common.utils.*;
+import org.apache.rocketmq.streams.configurable.model.Configure;
+
+import java.util.*;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public abstract class AbstractConfigurableService implements IConfigurableService {
+
+ private static final Log LOG = LogFactory.getLog(AbstractConfigurableService.class);
+
+ private static final String CLASS_NAME = IConfigurableService.CLASS_NAME;
+
+ protected Map<String, List<IConfigurable>> type2ConfigurableMap = new HashMap<>();
+
+ protected Map<String, IConfigurable> name2ConfigurableMap = new HashMap<>();
+
+ protected Map<String, IConfigurable> configurableMap = new HashMap<>();
+
+ protected Properties properties;
+
+ protected transient String namespace;
+
+ public AbstractConfigurableService(Properties properties) {
+ this.properties = properties;
+ }
+
+ public AbstractConfigurableService() {
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String identification) {
+ return name2ConfigurableMap.get(identification);
+ }
+
+ protected String getConfigureKey(String nameSpace, String type, String name) {
+ return MapKeyUtil.createKey(nameSpace, type, name);
+ }
+
+ protected void updateConfiguresCache(IConfigurable configurable) {
+ if (configurable == null) {
+ return;
+ }
+ configurable.toJson();
+ String key = getConfigureKey(configurable.getNameSpace(), configurable.getType(), configurable.getConfigureName());
+ configurableMap.put(key, configurable);
+ }
+
+ protected void updateConfiguresCache(List<IConfigurable> configureList) {
+ for (IConfigurable iConfigurable : configureList) {
+ updateConfiguresCache(iConfigurable);
+ }
+ }
+
+ protected boolean equals(String key, List<?> newConfigureList) {
+ for (Object o : newConfigureList) {
+ IConfigurable configure = (IConfigurable)o;
+ String tempKey = getConfigureKey(configure.getNameSpace(), configure.getType(), configure.getConfigureName());
+ if (key.equals(tempKey)) {
+ IConfigurable oldConfigure = configurableMap.get(key);
+ if (oldConfigure == null) {
+ continue;
+ }
+ return ConfigurableUtil.compare(oldConfigure, configure);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> queryConfigurableByType(String type) {
+ List<IConfigurable> list = queryConfigurable(type);
+ if (list == null) {
+ return new ArrayList<T>();
+ }
+ List<T> result = new ArrayList<T>();
+ for (IConfigurable configurable : list) {
+ result.add((T)configurable);
+ }
+ return result;
+ }
+
+ @Override
+ public boolean refreshConfigurable(String namespace) {
+ //每次刷新，重新刷新配置文件
+ //if(ComponentCreator.propertiesPath!=null){
+ // ComponentCreator.setProperties(ComponentCreator.propertiesPath);
+ //}
+ this.namespace = namespace;
+ // Map<String, List<IConfigurable>> namespace2ConfigurableMap = new HashMap<>();
+ Map<String, List<IConfigurable>> tempType2ConfigurableMap = new HashMap<>();
+ Map<String, IConfigurable> tempName2ConfigurableMap = new HashMap<>();
+ GetConfigureResult configures = loadConfigurable(namespace);
+ // updateConfiguresCache(configures.getConfigure());
+ if (configures != null && configures.isQuerySuccess() && configures.getConfigurables() != null) {
+ // List<Configure> configureList = filterConfigure(configures.getConfigure());
+ List<IConfigurable> configurables = configures.getConfigurables();
+ List<IConfigurable> configurableList = checkAndUpdateConfigurables(namespace, configurables,
+ tempType2ConfigurableMap, tempName2ConfigurableMap,
+ configures.getConfigurables());
+ // this.namespace2ConfigurableMap = namespace2ConfigurableMap;
+ for (IConfigurable configurable : configurableList) {
+ if (configurable instanceof IAfterConfiguableRefreshListerner) {
+ ((IAfterConfiguableRefreshListerner)configurable).doProcessAfterRefreshConfigurable(this);
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public <T> T queryConfigurable(String configurableType, String name) {
+ return (T)queryConfigurableByIdent(configurableType, name);
+ }
+
+ protected List<IConfigurable> checkAndUpdateConfigurables(String namespace, List<IConfigurable> configurables,
+ Map<String, List<IConfigurable>> tempType2ConfigurableMap,
+ Map<String, IConfigurable> tempName2ConfigurableMap,
+ List configures) {
+ List<IConfigurable> configurableList = new ArrayList<>();
+ for (IConfigurable configurable : configurables) {
+ try {
+ boolean isUpdate = update(configurable, tempName2ConfigurableMap, tempType2ConfigurableMap);
+ if (isUpdate) {
+ configurableList.add(configurable);
+ }
+ } catch (Exception e) {
+ LOG.error("组件初始化异常：" + e.getMessage() + ",name=" + configurable.getConfigureName(), e);
+ }
+ }
+ destroyOldConfigurables(tempName2ConfigurableMap);
+ this.name2ConfigurableMap = tempName2ConfigurableMap;
+ this.type2ConfigurableMap = tempType2ConfigurableMap;
+ return configurableList;
+ }
+
+ private void destroyOldConfigurables(Map<String, IConfigurable> tempName2ConfigurableMap) {
+ Iterator<Map.Entry<String, IConfigurable>> it = this.name2ConfigurableMap.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, IConfigurable> entry = it.next();
+ String key = entry.getKey();
+ IConfigurable value = entry.getValue();
+ if (!tempName2ConfigurableMap.containsKey(key)) {
+ destroyOldConfigurable(value);
+ }
+ }
+
+ }
+
+ private void destroyOldConfigurable(IConfigurable oldConfigurable) {
+ if (AbstractConfigurable.class.isInstance(oldConfigurable)) {
+ ((AbstractConfigurable)oldConfigurable).destroy();
+ }
+ String key = getConfigureKey(oldConfigurable.getNameSpace(), oldConfigurable.getType(),
+ oldConfigurable.getConfigureName());
+ configurableMap.remove(key);
+ }
+
+ protected void initConfigurable(IConfigurable configurable) {
+ if (AbstractConfigurable.class.isInstance(configurable)) {
+ AbstractConfigurable abstractConfigurable = (AbstractConfigurable)configurable;
+ abstractConfigurable.setConfigurableService(this);
+ }
+
+ configurable.init();
+
+ }
+
+ /**
+ * 内部使用
+ */
+ private ScheduledExecutorService scheduledExecutorService;
+
+ @Override
+ public void initConfigurables(final String namespace) {
+ refreshConfigurable(namespace);
+ long polingTime = -1;
+ if (this.properties != null) {
+ String pollingTimeStr = this.properties.getProperty(AbstractComponent.POLLING_TIME);
+ if (StringUtil.isNotEmpty(pollingTimeStr)) {
+ polingTime = Long.valueOf(pollingTimeStr);
+ }
+ }
+ if (polingTime > 0) {
+ scheduledExecutorService = new ScheduledThreadPoolExecutor(3);
+ scheduledExecutorService.scheduleWithFixedDelay(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ refreshConfigurable(namespace);
+ } catch (Exception e) {
+ LOG.error("Load configurables error:" + e.getMessage(), e);
+ }
+ }
+ }, polingTime, polingTime, TimeUnit.SECONDS);
+ }
+ }
+ // @Override
+ // public List<IConfigurable> queryConfigurable(String nameSpace) {
+ // return namespace2ConfigurableMap.get(nameSpace);
+ // }
+
+ @Override
+ public List<IConfigurable> queryConfigurable(String type) {
+ String key = MapKeyUtil.createKey(type);
+ return type2ConfigurableMap.get(key);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String type, String name) {
+ String key = MapKeyUtil.createKey(type, name);
+ return name2ConfigurableMap.get(key);
+ }
+
+ /**
+ * 根据namespace加载配置信息
+ *
+ * @param namespace
+ * @return
+ * @throws Exception
+ */
+ protected abstract GetConfigureResult loadConfigurable(String namespace);
+
+ @Override
+ public void update(IConfigurable configurable) {
+ // update(configurable,name2ConfigurableMap,type2ConfigurableMap);
+ updateConfigurable(configurable);
+ }
+
+ protected abstract void updateConfigurable(IConfigurable configurable);
+
+ protected abstract void insertConfigurable(IConfigurable configurable);
+
+ protected boolean update(IConfigurable configurable, Map<String, IConfigurable> name2ConfigurableMap,
+ Map<String, List<IConfigurable>> type2ConfigurableMap) {
+ if (configurable == null) {
+ return false;
+ }
+
+ boolean isUpdate = false;
+ List<IConfigurable> configurableList = new ArrayList<>();
+ configurableList.add(configurable);
+
+ String nameKey = MapKeyUtil.createKey(configurable.getType(), configurable.getConfigureName());
+ if (this.name2ConfigurableMap.containsKey(nameKey)) {
+ String configureKey = getConfigureKey(namespace, configurable.getType(), configurable.getConfigureName());
+ IConfigurable oldConfigurable = this.name2ConfigurableMap.get(nameKey);
+ if (equals(configureKey, configurableList)) {
+ configurable = oldConfigurable;
+ // name2ConfigurableMap.put(nameKey, name2ConfigurableMap.get(nameKey));
+ } else {
+ destroyOldConfigurable(oldConfigurable);
+ initConfigurable(configurable);
+ isUpdate = true;
+ }
+ } else {
+ initConfigurable(configurable);
+ isUpdate = true;
+ }
+ updateConfiguresCache(configurable);
+ name2ConfigurableMap.put(nameKey, configurable);
+ String typeKey = MapKeyUtil.createKey(configurable.getType());
+ // put2Map(namespace2ConfigurableMap, namespace, configurable);
+ put2Map(type2ConfigurableMap, typeKey, configurable);
+ return isUpdate;
+ }
+
+ @Override
+ public void insert(IConfigurable configurable) {
+ // update(configurable,name2ConfigurableMap,type2ConfigurableMap);
+ insertConfigurable(configurable);
+ }
+
+ /**
+ * 给一个扣，可以跨命名空间查询数据
+ *
+ * @param namespaces
+ * @return
+ */
+ public List<IConfigurable> queryConfiguableByNamespace(String... namespaces) {
+ List<IConfigurable> configurables = new ArrayList<>();
+ if (namespaces == null || namespaces.length == 0) {
+ return configurables;
+ }
+ for (String namespace : namespaces) {
+ GetConfigureResult result = loadConfigurable(namespace);
+ if (result.querySuccess) {
+ if (result.configurables != null && result.configurables.size() > 0) {
+ configurables.addAll(result.configurables);
+ }
+ } else {
+ throw new RuntimeException("Load configurable error, the namespace is " + namespace);
+ }
+ }
+ return configurables;
+
+ }
+
+ /**
+ * 往一个value是list的map中添加数据，如果list是空创建，否则直接插入
+ *
+ * @param map
+ * @param key
+ * @param configurable
+ */
+ protected void put2Map(Map<String, List<IConfigurable>> map, String key, IConfigurable configurable) {
+ List<IConfigurable> list = map.get(key);
+ if (list == null) {
+ list = new ArrayList<IConfigurable>();
+ map.put(key, list);
+ }
+ list.add(configurable);
+ }
+
+ @Override
+ public Collection<IConfigurable> findAll() {
+ return name2ConfigurableMap.values();
+ }
+
+ /**
+ * 把configurable转换成configure
+ *
+ * @param configurable
+ * @return
+ */
+ protected Configure createConfigure(IConfigurable configurable) {
+ Configure configure = new Configure();
+ configure.setType(configurable.getType());
+ configure.setName(configurable.getConfigureName());
+ configure.setNameSpace(configurable.getNameSpace());
+ String jsonString = configurable.toJson();
+ if (!StringUtil.isEmpty(jsonString)) {
+ JSONObject jsonObject = JSONObject.parseObject(jsonString);
+ jsonObject.put(CLASS_NAME, configurable.getClass().getName());
+ configure.setJsonValue(jsonObject.toJSONString());
+ }
+ // configure.createIdentification();
+ return configure;
+ }
+
+ @Override
+ public <T> Map<String, T> queryConfigurableMapByType(String type) {
+ List<IConfigurable> configurables = queryConfigurable(type);
+ if (configurables == null) {
+ return new HashMap<String, T>();
+ }
+ Map<String, T> result = new HashMap<String, T>();
+ for (IConfigurable configurable : configurables) {
+ result.put(configurable.getConfigureName(), (T)configurable);
+ }
+ return result;
+ }
+
+ /**
+ * 把configure转换成configurable
+ *
+ * @param configures
+ * @return
+ */
+ protected List<IConfigurable> convert(List<Configure> configures) {
+ if (configures == null) {
+ return new ArrayList<IConfigurable>();
+ }
+ List<IConfigurable> configurables = new ArrayList<IConfigurable>();
+ for (Configure configure : configures) {
+ IConfigurable configurable = convert(configure);
+ if (configurable != null) {
+ configurables.add(configurable);
+ }
+
+ }
+ return configurables;
+ }
+
+ protected IConfigurable createConfigurableFromJson(String namespace, String type, String name, String jsonValue) {
+ if (StringUtil.isEmpty(jsonValue)) {
+ return null;
+ }
+ JSONObject jsonObject = JSONObject.parseObject(jsonValue);
+ String className = jsonObject.getString(CLASS_NAME);
+ IConfigurable configurable = createConfigurable(className);
+ if (configurable == null) {
+ return null;
+ }
+ configurable.setConfigureName(name);
+ configurable.setNameSpace(namespace);
+ configurable.setType(type);
+ if (AbstractConfigurable.class.isInstance(configurable)) {
+ AbstractConfigurable abstractConfigurable = (AbstractConfigurable)configurable;
+ abstractConfigurable.setConfigurableService(this);
+ }
+ configurable.toObject(jsonValue);
+ return configurable;
+ }
+
+ /**
+ * 提供一个入口，可以让外部用户改变configure对应的configurable的值
+ *
+ * @param configure
+ * @return
+ */
+ protected IConfigurable convert(Configure configure) {
+
+ return convertConfigurable(configure);
+ }
+
+ protected IConfigurable convertConfigurable(Configure configure) {
+ String className = null;
+ try {
+ String jsonString = configure.getJsonValue();
+ IConfigurable configurable =
+ createConfigurableFromJson(configure.getNameSpace(), configure.getType(), configure.getName(),
+ jsonString);
+ if (configurable instanceof Entity) {
+ // add by wangtl 20171110 Configurable接口第三方包也在用，故不能Configurable里加接口，只能加到抽象类里，这里强转下
+ Entity abs = (Entity)configurable;
+ abs.setId(configure.getId());
+ abs.setGmtCreate(configure.getGmtCreate());
+ abs.setGmtModified(configure.getGmtModified());
+ /*
+ * abs.setTempKey((configurable.getNameSpace() + configurable.getType() +
+ * configurable.getConfigureName() + jsonString).hashCode());
+ */
+ }
+ convertPost(configurable);
+ return configurable;
+ } catch (Exception e) {
+ LOG.error("转换异常：" + configure.toString(), e);
+ return null;
+ }
+ }
+
+ /**
+ * 如果需要改变configurable的属性，可以再这里设置
+ *
+ * @param configurable
+ */
+ @SuppressWarnings("rawtypes")
+ protected void convertPost(IConfigurable configurable) {
+ if (this.properties == null) {
+ return;
+ }
+ String identification =
+ MapKeyUtil.createKey(configurable.getNameSpace(), configurable.getType(), configurable.getConfigureName());
+ String propertyValue = this.properties.getProperty(identification);
+ if (StringUtil.isEmpty(propertyValue)) {
+ return;
+ }
+ String[] fieldName2Values = propertyValue.split(",");
+ if (fieldName2Values == null || fieldName2Values.length == 0) {
+ return;
+ }
+ for (String fieldName2Value : fieldName2Values) {
+ try {
+ String[] fieldName2ValueArray = fieldName2Value.split(":");
+ if (fieldName2ValueArray == null || fieldName2ValueArray.length != 2) {
+ continue;
+ }
+ String fieldName = fieldName2ValueArray[0];
+ String value = fieldName2ValueArray[1];
+ Class clazz = ReflectUtil.getBeanFieldType(configurable.getClass(), fieldName);
+ DataType dataType = DataTypeUtil.createDataType(clazz, null);
+ if (dataType == null) {
+ continue;
+ }
+ Object fieldValue = dataType.getData(value);
+ ReflectUtil.setBeanFieldValue(configurable, fieldName, fieldValue);
+
+ } catch (Exception e) {
+ LOG.error("convert post error " + fieldName2Value, e);
+ continue;
+ }
+
+ }
+ }
+
+ /**
+ * 创建configurable对象
+ *
+ * @param className class name
+ * @return
+ */
+ @SuppressWarnings("rawtypes")
+ protected IConfigurable createConfigurable(String className) {
+ return ReflectUtil.forInstance(className);
+ }
+
+ public class GetConfigureResult {
+
+ private boolean querySuccess;
+ private List<IConfigurable> configurables;
+
+ public boolean isQuerySuccess() {
+ return querySuccess;
+ }
+
+ public void setQuerySuccess(boolean querySuccess) {
+ this.querySuccess = querySuccess;
+ }
+
+ public List<IConfigurable> getConfigurables() {
+ return configurables;
+ }
+
+ public void setConfigurables(List<IConfigurable> configurables) {
+ this.configurables = configurables;
+ }
+ }
+
+ @Override
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractSupportParentConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractSupportParentConfigureService.java
new file mode 100644
index 0000000..d2d26cd
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/AbstractSupportParentConfigureService.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * namespace 分层，支持顶级命名空间，顶级命名空间的对象，所有命名空间都可见。顶级命名空间是固定值IConfigurableService.PARENT_CHANNEL_NAME_SPACE
+ */
+public abstract class AbstractSupportParentConfigureService extends AbstractConfigurableService
+ implements IConfigurableService {
+
+ private static final Log LOG = LogFactory.getLog(AbstractSupportParentConfigureService.class);
+ protected IConfigurableService configureService = null;
+ protected IConfigurableService parentConfigureService = null;
+ //protected IConfigurableService shareConfigureService = null;
+ protected Properties properties;
+
+ public AbstractSupportParentConfigureService() {
+ super(null);
+ }
+
+ public void initMethod(Properties property) {
+ this.properties = property;
+ initBeforeInitConfigurable(property);
+ }
+
+ protected abstract void initBeforeInitConfigurable(Properties property);
+
+ @Override
+ public void initConfigurables(String namespace) {
+
+ if (!IConfigurableService.PARENT_CHANNEL_NAME_SPACE.equals(namespace)) {
+ parentConfigureService.initConfigurables(IConfigurableService.PARENT_CHANNEL_NAME_SPACE);
+ } else {
+ parentConfigureService = null;
+ }
+ configureService.initConfigurables(namespace);
+ }
+
+ @Override
+ public boolean refreshConfigurable(String namespace) {
+
+ if (!IConfigurableService.PARENT_CHANNEL_NAME_SPACE.equals(namespace)) {
+ parentConfigureService.refreshConfigurable(IConfigurableService.PARENT_CHANNEL_NAME_SPACE);
+ // initShareConfigurableService(namespace);
+ }
+ configureService.refreshConfigurable(namespace);
+ return true;
+ }
+
+ @Override
+ public List<IConfigurable> queryConfigurable(String type) {
+ List<IConfigurable> result = configureService.queryConfigurable(type);
+ if (result == null) {
+ result = new ArrayList<>();
+ }
+ //if (shareConfigureService != null) {
+ // List<IConfigurable> share = shareConfigureService.queryConfigurable(type);
+ // if (share != null) {
+ // result.addAll(share);
+ // }
+ //}
+ if (parentConfigureService == null) {
+ return result;
+ }
+ List<IConfigurable> parent = parentConfigureService.queryConfigurable(type);
+ if (parent != null) {
+ result.addAll(parent);
+ }
+ return result;
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String type, String name) {
+ IConfigurable configurable = configureService.queryConfigurableByIdent(type, name);
+ if (configurable != null) {
+ return configurable;
+ }
+ if (parentConfigureService == null) {
+ return null;
+ }
+ //if (shareConfigureService != null) {
+ // configurable = shareConfigureService.queryConfigurableByIdent(type, name);
+ //}
+ if (configurable != null) {
+ return configurable;
+ }
+ return parentConfigureService.queryConfigurableByIdent(type, name);
+ }
+
+ @Override
+ public IConfigurable queryConfigurableByIdent(String identification) {
+ IConfigurable configurable = configureService.queryConfigurableByIdent(identification);
+ if (configurable != null) {
+ return configurable;
+ }
+ if (parentConfigureService == null) {
+ return null;
+ }
+ //if (shareConfigureService != null) {
+ // configurable = shareConfigureService.queryConfigurableByIdent(identification);
+ //}
+ if (configurable != null) {
+ return configurable;
+ }
+ return parentConfigureService.queryConfigurableByIdent(identification);
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configurable) {
+ if (parentConfigureService != null && configurable.getNameSpace()
+ .equals(IConfigurableService.PARENT_CHANNEL_NAME_SPACE)) {
+ parentConfigureService.insert(configurable);
+ } else {
+ configureService.insert(configurable);
+ }
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configurable) {
+ if (parentConfigureService != null && configurable.getNameSpace()
+ .equals(IConfigurableService.PARENT_CHANNEL_NAME_SPACE)) {
+ parentConfigureService.update(configurable);
+ } else {
+ configureService.update(configurable);
+ }
+ }
+
+ @Override
+ public <T> T queryConfigurable(String configurableType, String name) {
+ return (T)queryConfigurableByIdent(configurableType, name);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ return null;
+ }
+
+ //protected void initShareConfigurableService(String namespace) {
+ // if (parentConfigureService == null) {
+ // return;
+ // }
+ // shareConfigureService = new AbstractReadOnlyConfigurableService() {
+ //
+ // @Override
+ // public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ // refreshConfigurable(namespace);
+ // return queryConfigurableByType(type);
+ // }
+ //
+ // @Override
+ // protected List<IConfigurable> loadConfigurables(String namespace) {
+ // List<IConfigurable> parent = parentConfigureService.queryConfigurable(ShareConfiguable.TYPE);
+ // List<IConfigurable> shareConfigurables = new ArrayList<>();
+ // if (parent == null) {
+ // return shareConfigurables;
+ // }
+ // for (IConfigurable configurable : parent) {
+ // ShareConfiguable shareConfiguable = (ShareConfiguable) configurable;
+ // if (shareConfiguable.getShareAll() || shareConfiguable.getShareNameSpaces().contains(namespace)) {
+ // String sharedNameSpace = shareConfiguable.getSharedNameSpace();
+ // String sharedType = shareConfiguable.getSharedType();
+ // String sharedName = shareConfiguable.getSharedName();
+ // List<IConfigurable> sharedConfigrables =
+ // createAndQueryConfigurable(sharedNameSpace, sharedType, sharedName);
+ // if (sharedConfigrables != null) {
+ // shareConfigurables.addAll(sharedConfigrables);
+ // }
+ // }
+ // }
+ // return shareConfigurables;
+ // }
+ //
+ //
+ // };
+ // shareConfigureService.refreshConfigurable(namespace);
+ //
+ //}
+
+ protected List<IConfigurable> createAndQueryConfigurable(String sharedNameSpace, String sharedType,
+ String sharedName) {
+ IConfigurableService innerSharedConfigurableService =
+ ConfigurableServiceFactory.createConfigurableService(properties);
+ innerSharedConfigurableService.refreshConfigurable(sharedNameSpace);
+ if (StringUtil.isNotEmpty(sharedName)) {
+ List<IConfigurable> configurables = new ArrayList<>();
+ IConfigurable configurable = innerSharedConfigurableService.queryConfigurableByIdent(sharedType, sharedName);
+ configurables.add(configurable);
+ return configurables;
+ } else {
+ return innerSharedConfigurableService.queryConfigurable(sharedType);
+ }
+
+ }
+
+ @Override
+ public Collection<IConfigurable> findAll() {
+ List<IConfigurable> configurables = new ArrayList<>();
+ if (parentConfigureService != null) {
+ Collection<IConfigurable> tmp = parentConfigureService.findAll();
+ if (tmp != null || tmp.size() > 0) {
+ configurables.addAll(tmp);
+ }
+ }
+ Collection<IConfigurable> tmp = configureService.findAll();
+ if (tmp != null || tmp.size() > 0) {
+ configurables.addAll(tmp);
+ }
+ return configurables;
+ }
+
+ public IConfigurableService getConfigureService() {
+ return configureService;
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ List<T> configurables = new ArrayList<>();
+ if (parentConfigureService != null) {
+ Collection<T> tmp = parentConfigureService.loadConfigurableFromStorage(type);
+ if (tmp != null || tmp.size() > 0) {
+ configurables.addAll(tmp);
+ }
+ }
+ Collection<T> tmp = configureService.loadConfigurableFromStorage(type);
+ if (tmp != null || tmp.size() > 0) {
+ configurables.addAll(tmp);
+ }
+ return configurables;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServcieType.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServcieType.java
new file mode 100644
index 0000000..a4998ff
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServcieType.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+
+@Deprecated
+public class ConfigurableServcieType {
+
+ public static final String DEFAULT_SERVICE_NAME = IConfigurableService.DEFAULT_SERVICE_NAME;
+ public static final String MEMORY_SERVICE_NAME = IConfigurableService.MEMORY_SERVICE_NAME;
+ public static final String FILE_SERVICE_NAME = IConfigurableService.FILE_SERVICE_NAME;
+ public static final String HTTP_SERVICE_NAME = IConfigurableService.HTTP_SERVICE_NAME;
+
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServiceFactory.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServiceFactory.java
new file mode 100644
index 0000000..449f04a
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/ConfigurableServiceFactory.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.serviceloader.ServiceLoaderComponent;
+
+import java.util.Properties;
+
+public class ConfigurableServiceFactory {
+ private static ServiceLoaderComponent<IConfigurableService> configurableServiceLoaderComponent =
+ ServiceLoaderComponent.getInstance(IConfigurableService.class);
+ public static final String CONFIGURABLE_SERVICE_TYPE = "dipper.configurable.service.type";
+ private static final Log LOG = LogFactory.getLog(ConfigurableServiceFactory.class);
+
+ public static IConfigurableService createConfigurableService(Properties properties) {
+ try {
+ Properties properties1 = new Properties();
+ properties1.putAll(properties);
+ String type = properties1.getProperty(CONFIGURABLE_SERVICE_TYPE);
+ if (StringUtil.isEmpty(type)) {
+ type = IConfigurableService.DEFAULT_SERVICE_NAME;
+ ;
+ }
+ IConfigurableService configurableService = getConfigurableServcieType(type);
+ if (AbstractSupportParentConfigureService.class.isInstance(configurableService)) {
+ ((AbstractSupportParentConfigureService)configurableService).initMethod(properties1);
+ }
+ return configurableService;
+ } catch (Exception e) {
+ LOG.error("create ConfigurableService error", e);
+ return null;
+ }
+
+ }
+
+ public static IConfigurableService getConfigurableServcieType(String type) {
+ IConfigurableService configurableService = (IConfigurableService)configurableServiceLoaderComponent.getService().loadService(type);
+ return ReflectUtil.forInstance(configurableService.getClass().getName());
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileConfigureService.java
new file mode 100644
index 0000000..1a5fe9b
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileConfigureService.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service.impl;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configurable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.configurable.model.Configure;
+import org.apache.rocketmq.streams.common.model.Entity;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+public class FileConfigureService extends AbstractConfigurableService {
+
+ public static final String FILE_PATH_NAME = IConfigurableService.FILE_PATH_NAME;
+ // 配置文件的路径
+ private static final Log LOG = LogFactory.getLog(FileConfigureService.class);
+ private static final String DEFAULT_FILE_NAME = "dipper_configure.cs"; // 默认文件名
+ private static final String SIGN = "&&&&"; // 字段分割附号
+ public String fileName;
+
+ public FileConfigureService(Properties properties) {
+ super(properties);
+ initService(properties.getProperty(FILE_PATH_NAME));
+ }
+
+ protected void initService(String fileAndPath) {
+ if (StringUtil.isEmpty(fileAndPath)) {
+ String path = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
+ if (path.endsWith(".jar")) {
+ int index = path.lastIndexOf(File.separator);
+ path = path.substring(0, index);
+ }
+ fileName = FileUtil.concatFilePath(path, DEFAULT_FILE_NAME);
+ } else {
+ fileName = fileAndPath;
+ }
+ LOG.info("load file from path = " + fileName);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ GetConfigureResult result = new GetConfigureResult();
+ try {
+ List<Configure> configures = selectOpening(namespace);
+ List<IConfigurable> configurables = convert(configures);
+ LOG.info("load configure namespace=" + namespace + " count=" + configures.size());
+ result.setConfigurables(configurables);
+ result.setQuerySuccess(true);// 该字段标示查询是否成功，若不成功则不会更新配置
+ } catch (Exception e) {
+ result.setQuerySuccess(false);
+ e.printStackTrace();
+ LOG.error("load configurable error ", e);
+ }
+ return result;
+ }
+
+ protected List<Configure> selectOpening(String namespace) {
+ List<String> list = loadFileLine(fileName);
+ List<Configure> configures = convert2Configure(list);
+ return filter(configures, namespace);
+ }
+
+ protected List<Configure> filter(List<Configure> configures, String namespace) {
+ if (configures == null) {
+ return new ArrayList<>();
+ }
+ if (StringUtil.isEmpty(namespace)) {
+ throw new RuntimeException("namespace can not empty ");
+ }
+ List<Configure> filterConfigures = new ArrayList<>();
+ for (Configure configure : configures) {
+ if (!namespace.equals(configure.getNameSpace())) {
+ continue;
+ }
+ filterConfigures.add(configure);
+ }
+ return filterConfigures;
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configure) {
+ if (configure == null) {
+ LOG.warn("insert configure is null");
+ return;
+ }
+ String row = configure2String(configure);
+
+ List<String> rows = loadFileLine(fileName);
+ if (rows == null) {
+ rows = new ArrayList<>();
+ }
+ List<Configure> configures = convert2Configure(rows);
+ String newKey =
+ MapKeyUtil.createKey(configure.getNameSpace(), configure.getType(), configure.getConfigureName());
+ boolean isReplace = false;
+ for (int i = 0; i < configures.size(); i++) {
+ Configure c = configures.get(i);
+ String old = MapKeyUtil.createKey(c.getNameSpace(), c.getType(), c.getName());
+ if (old.equals(newKey)) {
+ rows.set(i, configure2String(configure));
+ isReplace = true;
+ break;
+ }
+ }
+ if (!isReplace) {
+ rows.add(configure2String(configure));
+ }
+ writeFile(fileName, rows);
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configure) {
+ if (configure == null) {
+ LOG.warn("insert configure is null");
+ return;
+ }
+
+ List<String> rows = FileUtil.loadFileLine(fileName);
+ if (rows == null) {
+ rows = new ArrayList<>();
+ }
+ for (int i = 0; i < rows.size(); i++) {
+ String row = rows.get(i);
+ Configure oldConfigure = convert(row);
+ if (configure.getNameSpace().equals(oldConfigure.getNameSpace()) && configure.getType()
+ .equals(oldConfigure.getType()) && configure.getConfigureName().equals(oldConfigure.getName())) {
+ rows.set(i, configure2String(configure));
+ }
+ }
+ writeFile(fileName, rows);
+
+ }
+
+ protected Configure convert(String row) {
+ String[] values = row.split(SIGN);
+ String namespace = getColumnValue(values, 0, "namespace");
+ String type = getColumnValue(values, 1, "type");
+ String name = getColumnValue(values, 2, "name");
+ String jsonValue = getColumnValue(values, 3, "json_value");
+ String createDate = getColumnValue(values, 4, "gmt_create");
+ String modifiedDate = getColumnValue(values, 5, "gmt_modified");
+ String id = getColumnValue(values, 6, "id");
+ Configure configure = new Configure();
+ configure.setNameSpace(namespace);
+ configure.setType(type);
+ configure.setName(name);
+ configure.setJsonValue(jsonValue);
+ configure.setGmtCreate(DateUtil.parse(createDate));
+ configure.setGmtCreate(DateUtil.parse(modifiedDate));
+ configure.setId((id == null ? null : Long.valueOf(id)));
+
+ return configure;
+ }
+
+ protected List<Configure> convert2Configure(List<String> rows) {
+ List<Configure> configures = new ArrayList<Configure>();
+ for (String row : rows) {
+ configures.add(convert(row));
+ }
+ return configures;
+ }
+
+ protected String getColumnValue(String[] values, int i, String namespace) {
+ if (values == null || values.length == 0) {
+ return null;
+ }
+ if (values.length <= i) {
+ return null;
+ }
+ if ("null".equals(values[i])) {
+ return null;
+ }
+ return values[i];
+ }
+
+ /**
+ * 解密文件，并加载到内存
+ *
+ * @param fileName
+ * @return
+ */
+ protected List<String> loadFileLine(String fileName) {
+ List<String> rows = FileUtil.loadFileLine(fileName);
+ if (rows == null) {
+ rows = new ArrayList<>();
+ }
+ return doDecRowList(rows);
+ }
+
+ protected void writeFile(String fileName, List<String> rows) {
+ List<String> rowList = doEncryptRowList(rows);
+ FileUtil.write(fileName, rowList);
+ }
+
+ private List<String> doEncryptRowList(List<String> rows) {
+ return rows;
+ }
+
+ private List<String> doDecRowList(List<String> rows) {
+ return rows;
+ }
+
+ protected String configure2String(IConfigurable configure) {
+ Entity entity = null;
+ if (configure instanceof Entity) {
+ entity = (Entity)configure;
+ } else {
+ entity = new Entity();
+ }
+ String row = MapKeyUtil.createKeyBySign(SIGN, configure.getNameSpace(), configure.getType(),
+ configure.getConfigureName(), configure.toJson(), DateUtil.format(entity.getGmtCreate()),
+ DateUtil.format(entity.getGmtModified()), entity.getId() + "");
+ return row;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ refreshConfigurable(getNamespace());
+ return queryConfigurableByType(type);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileSupportParentConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileSupportParentConfigureService.java
new file mode 100644
index 0000000..aafa805
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/FileSupportParentConfigureService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service.impl;
+
+import org.apache.rocketmq.streams.configurable.service.AbstractSupportParentConfigureService;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import com.google.auto.service.AutoService;
+
+import java.util.Properties;
+
+@AutoService(IConfigurableService.class)
+@ServiceName(ConfigurableServcieType.FILE_SERVICE_NAME)
+public class FileSupportParentConfigureService extends AbstractSupportParentConfigureService {
+
+ @Override
+ protected void initBeforeInitConfigurable(Properties property) {
+ this.configureService = new FileConfigureService(properties);
+ this.parentConfigureService = new FileConfigureService(properties);
+ }
+
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemoryConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemoryConfigureService.java
new file mode 100644
index 0000000..b6d1fd9
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemoryConfigureService.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.configurable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+
+public class MemoryConfigureService extends AbstractConfigurableService {
+
+ private static Map<String, List<IConfigurable>> namespace2Configure = new HashMap<>();
+
+ public MemoryConfigureService(Properties properties) {
+ super(properties);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ GetConfigureResult result = new GetConfigureResult();
+ result.setQuerySuccess(true);
+ List<IConfigurable> configurableList = new ArrayList<>();
+ List<IConfigurable> configurables = namespace2Configure.get(namespace);
+ if (configurables == null) {
+ configurableList = null;
+ } else {
+ List<IConfigurable> tmps = new ArrayList<>();
+ tmps.addAll(configurables);
+ for (IConfigurable configurable : tmps) {
+ IConfigurable tmp = ReflectUtil.forInstance(configurable.getClass());
+ tmp.toObject(configurable.toJson());
+ tmp.setNameSpace(configurable.getNameSpace());
+ tmp.setConfigureName(configurable.getConfigureName());
+ configurableList.add(tmp);
+ }
+ }
+ result.setConfigurables(configurableList);
+ return result;
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configurable) {
+ if (configurable == null) {
+ return;
+ }
+
+ String namespace = configurable.getNameSpace();
+ List<IConfigurable> list = namespace2Configure.get(namespace);
+ if (list == null) {
+ synchronized (this) {
+ list = namespace2Configure.get(namespace);
+ if (list == null) {
+ list = new ArrayList<>();
+ namespace2Configure.put(namespace, list);
+ }
+ }
+ }
+ int removeIndex = -1;
+ for (int i = 0; i < list.size(); i++) {
+ IConfigurable config = list.get(i);
+ if (config.getType().equals(configurable.getType()) && config.getConfigureName()
+ .equals(configurable.getConfigureName())) {
+ removeIndex = i;
+ }
+ }
+ if (AbstractConfigurable.class.isInstance(configurable)) {
+ ((AbstractConfigurable)configurable).setConfigurableService(this);
+ }
+ if (removeIndex != -1) {
+ list.remove(removeIndex);
+ }
+ list.add(configurable);
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configure) {
+ List<IConfigurable> list = namespace2Configure.get(configure.getNameSpace());
+ if (list == null || list.size() == 0) {
+ throw new RuntimeException(
+ "not have exist configure " + configure.getNameSpace() + "," + configure.getType() + ","
+ + configure.getConfigureName());
+ }
+ for (int i = 0; i < list.size(); i++) {
+ IConfigurable config = list.get(i);
+ if (config.getType().equals(configure.getType()) && config.getConfigureName()
+ .equals(configure.getConfigureName())) {
+ list.set(i, configure);
+ return;
+ }
+ }
+ throw new RuntimeException(
+ "not have exist configure " + configure.getNameSpace() + "," + configure.getType() + ","
+ + configure.getConfigureName());
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+ refreshConfigurable(getNamespace());
+ return queryConfigurableByType(type);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemorySupportParentConfigureService.java b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemorySupportParentConfigureService.java
new file mode 100644
index 0000000..92d2095
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/java/org/apache/rocketmq/streams/configurable/service/impl/MemorySupportParentConfigureService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.service.impl;
+
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.configurable.service.AbstractSupportParentConfigureService;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import com.google.auto.service.AutoService;
+
+@AutoService(IConfigurableService.class)
+@ServiceName(ConfigurableServcieType.MEMORY_SERVICE_NAME)
+public class MemorySupportParentConfigureService extends AbstractSupportParentConfigureService {
+
+ @Override
+ protected void initBeforeInitConfigurable(Properties property) {
+ // this.rootConfigureService = new MemoryConfigureService(property);
+ this.parentConfigureService = new MemoryConfigureService(properties);
+ this.configureService = new MemoryConfigureService(properties);
+ }
+}
diff --git a/rocketmq-streams-configurable/src/main/resources/log4j.xml b/rocketmq-streams-configurable/src/main/resources/log4j.xml
new file mode 100755
index 0000000..7812fe7
--- /dev/null
+++ b/rocketmq-streams-configurable/src/main/resources/log4j.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
diff --git a/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/ConfiguableComponentTest.java b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/ConfiguableComponentTest.java
new file mode 100644
index 0000000..50cc83b
--- /dev/null
+++ b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/ConfiguableComponentTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable;
+
+import java.util.List;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configuable.model.Person;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+public class ConfiguableComponentTest {
+
+ @Test
+ public void testInsertConfiguable(){
+ String namespace="org.apache.configuable.test";
+ ConfigurableComponent configurableComponent= ConfigurableComponent.getInstance(namespace);
+ Person person=createPerson(namespace);
+ configurableComponent.insert(person);//完成数据存储，在配置文件配置存储类型，支持内存，db和文件，默认是内存
+ //查询只操作内存，存储的数据定时加载到内存，刚插入的数据，还未加载，查询不到
+ assertTrue(configurableComponent.queryConfigurable("person","personName")==null);
+ configurableComponent.refreshConfigurable(namespace);//强制加载数据到内存，可以查询数据
+ assertTrue(configurableComponent.queryConfigurable("person","peronName")!=null);
+ }
+
+
+ @Test
+ public void testConfiguableENVDependence(){
+ String namespace="org.apache.configuable.test";
+ ConfigurableComponent configurableComponent= ConfigurableComponent.getInstance(namespace);
+ Person person=createPerson(namespace);
+ person.setName("persion.name");//对于有ENVDependence的字段，可以不存储真值，存储一个key，把真值配置在配置文件中
+ configurableComponent.insert(person);//完成数据存储，在配置文件配置存储类型，支持内存，db和文件，默认是内存
+ ComponentCreator.getProperties().put("persion.name","realName");//这个代表真实的配置文件，启动时会把配置文件的内容加载到ComponentCreator.getProperties()中
+ configurableComponent.refreshConfigurable(namespace);//刷新存储
+ person=configurableComponent.queryConfigurable("person","peronName");
+ assertTrue(person.getName().equals("realName"));
+ }
+
+
+ @Test
+ public void testSupportParentNameSpace(){
+ String namespace="org.apache.configuable.test";
+ ConfigurableComponent configurableComponent= ConfigurableComponent.getInstance(namespace);
+ Person person=createPerson(namespace);
+ Person otherPerson=createPerson("org.apache.configuable.test1");
+ configurableComponent.insert(person);
+ configurableComponent.insert(otherPerson);
+ configurableComponent.refreshConfigurable(namespace);
+ //只加载自己命名空间的对象
+ List<Person> personList=configurableComponent.queryConfigurableByType("person");
+ assertTrue(personList.size()==1);
+
+ /**
+ * 顶级命名空间的对象，所有namespace都可见
+ */
+ Person thirdPerson=createPerson(IConfigurableService.PARENT_CHANNEL_NAME_SPACE);
+ configurableComponent.insert(thirdPerson);
+ configurableComponent.refreshConfigurable(namespace);//只加载自己命名空间的对象
+ personList=configurableComponent.queryConfigurableByType("person");
+ assertTrue(personList.size()==2);
+ }
+
+
+ //测试定时加载逻辑，当对象的updateFlag值变化后，才会被替换旧对象
+ @Test
+ public void testAutoLoader() throws InterruptedException {
+ ComponentCreator.getProperties().put(AbstractComponent.POLLING_TIME,"1");//1秒后动态加载对象
+ String namespace="org.apache.configuable.test";
+ ConfigurableComponent configurableComponent= ConfigurableComponent.getInstance(namespace);
+ Person person=createPerson(namespace);
+ configurableComponent.insert(person);
+ Thread.sleep(2000);//1秒后，新插入的对象会被加载
+ person=configurableComponent.queryConfigurable("person","peronName");
+ assertTrue(person!=null);
+
+
+ }
+
+ /**
+ * 创建configuable对象
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace){
+ Person person=new Person();
+ person.setName("chris");
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/model/Person.java b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/model/Person.java
new file mode 100644
index 0000000..8c4f076
--- /dev/null
+++ b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configuable/model/Person.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+public class Person extends BasedConfigurable{
+ @ENVDependence
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+
+ @Override
+ public Object clone() {
+ Person person = null;
+ try {
+ person = (Person)super.clone();
+ } catch (CloneNotSupportedException e) {
+ System.out.println("clone error " + e);
+ }
+ return person;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/ConfigurableComponentTest.java b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/ConfigurableComponentTest.java
new file mode 100644
index 0000000..6e632b1
--- /dev/null
+++ b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/ConfigurableComponentTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable;
+
+import java.util.List;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configurable.model.Person;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+public class ConfigurableComponentTest {
+
+ @Test
+ public void testInsertConfigurable() {
+ String namespace = "org.apache.configurable.test";
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ Person person = createPerson(namespace);
+ configurableComponent.insert(person);//完成数据存储，在配置文件配置存储类型，支持内存，db和文件，默认是内存
+ //查询只操作内存，存储的数据定时加载到内存，刚插入的数据，还未加载，查询不到
+ assertTrue(configurableComponent.queryConfigurable("person", "personName") == null);
+ configurableComponent.refreshConfigurable(namespace);//强制加载数据到内存，可以查询数据
+ assertTrue(configurableComponent.queryConfigurable("person", "peronName") != null);
+ }
+
+ @Test
+ public void testConfigurableENVDependence() {
+ String namespace = "org.apache.configurable.test";
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ Person person = createPerson(namespace);
+ person.setName("persion.name");//对于有ENVDependence的字段，可以不存储真值，存储一个key，把真值配置在配置文件中
+ configurableComponent.insert(person);//完成数据存储，在配置文件配置存储类型，支持内存，db和文件，默认是内存
+ ComponentCreator.getProperties().put("persion.name", "realName");//这个代表真实的配置文件，启动时会把配置文件的内容加载到ComponentCreator.getProperties()中
+ configurableComponent.refreshConfigurable(namespace);//刷新存储
+ person = configurableComponent.queryConfigurable("person", "peronName");
+ assertTrue(person.getName().equals("realName"));
+ }
+
+ @Test
+ public void testSupportParentNameSpace() {
+ String namespace = "org.apache.configurable.test";
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ Person person = createPerson(namespace);
+ Person otherPerson = createPerson("org.apache.configuable.test1");
+ configurableComponent.insert(person);
+ configurableComponent.insert(otherPerson);
+ configurableComponent.refreshConfigurable(namespace);
+ //只加载自己命名空间的对象
+ List<Person> personList = configurableComponent.queryConfigurableByType("person");
+ assertTrue(personList.size() == 1);
+
+ /**
+ * 顶级命名空间的对象，所有namespace都可见
+ */
+ Person thirdPerson = createPerson(IConfigurableService.PARENT_CHANNEL_NAME_SPACE);
+ configurableComponent.insert(thirdPerson);
+ configurableComponent.refreshConfigurable(namespace);//只加载自己命名空间的对象
+ personList = configurableComponent.queryConfigurableByType("person");
+ assertTrue(personList.size() == 2);
+ }
+
+ //测试定时加载逻辑，当对象的updateFlag值变化后，才会被替换旧对象
+ @Test
+ public void testAutoLoader() throws InterruptedException {
+ ComponentCreator.getProperties().put(AbstractComponent.POLLING_TIME, "1");//1秒后动态加载对象
+ String namespace = "org.apache.configurable.test";
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ Person person = createPerson(namespace);
+ configurableComponent.insert(person);
+ Thread.sleep(2000);//1秒后，新插入的对象会被加载
+ person = configurableComponent.queryConfigurable("person", "peronName");
+ assertTrue(person != null);
+
+ }
+
+ /**
+ * 创建configurable对象
+ *
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setName("chris");
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/model/Person.java b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/model/Person.java
new file mode 100644
index 0000000..06fd678
--- /dev/null
+++ b/rocketmq-streams-configurable/src/test/java/org/apache/rocketmq/streams/configurable/model/Person.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+
+public class Person extends BasedConfigurable {
+ @ENVDependence
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+
+ @Override
+ public Object clone() {
+ Person person = null;
+ try {
+ person = (Person)super.clone();
+ } catch (CloneNotSupportedException e) {
+ System.out.println("clone error " + e);
+ }
+ return person;
+ }
+}
diff --git a/rocketmq-streams-configurable/src/test/resources/log4j.xml b/rocketmq-streams-configurable/src/test/resources/log4j.xml
new file mode 100755
index 0000000..7812fe7
--- /dev/null
+++ b/rocketmq-streams-configurable/src/test/resources/log4j.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
[rocketmq-streams] 02/15: add lease、dim and client module
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit b757f8db8141d282dd793d37af9b509bcc54b7ca
Author: 刈刀 <ju...@alibaba-inc.com>
AuthorDate: Mon Aug 2 11:21:45 2021 +0800
add lease、dim and client module
---
.gitignore | 25 ++
README.md | 108 +++++
pom.xml | 410 +++++++++++++++++++
rocketmq-streams-clients/pom.xml | 46 +++
.../rocketmq/streams/client/DataStreamAction.java | 101 +++++
.../rocketmq/streams/client/StreamBuilder.java | 28 ++
.../streams/client/source/DataStreamSource.java | 76 ++++
.../client/strategy/CheckpointStrategy.java | 69 ++++
.../streams/client/strategy/StateStrategy.java | 37 ++
.../rocketmq/streams/client/strategy/Strategy.java | 25 ++
.../streams/client/transform/DataStream.java | 437 +++++++++++++++++++++
.../streams/client/transform/JoinStream.java | 212 ++++++++++
.../streams/client/transform/SplitStream.java | 61 +++
.../streams/client/transform/WindowStream.java | 210 ++++++++++
.../client/transform/window/HoppingWindow.java | 32 ++
.../client/transform/window/SessionWindow.java | 32 ++
.../streams/client/transform/window/Time.java | 45 +++
.../client/transform/window/TumblingWindow.java | 33 ++
.../client/transform/window/WindowInfo.java | 83 ++++
.../rocketmq/streams/client/DBDriverTest.java | 75 ++++
.../rocketmq/streams/client/DataStreamTest.java | 107 +++++
.../apache/rocketmq/streams/client/FilterTest.java | 49 +++
.../apache/rocketmq/streams/client/JoinTest.java | 89 +++++
.../apache/rocketmq/streams/client/LeaseTest.java | 98 +++++
.../rocketmq/streams/client/ORMUtilTest.java | 172 ++++++++
.../apache/rocketmq/streams/client/SplitTest.java | 86 ++++
.../apache/rocketmq/streams/client/UnionTest.java | 82 ++++
.../apache/rocketmq/streams/client/WindowTest.java | 86 ++++
.../client/windows/AbstractWindowFireModeTest.java | 189 +++++++++
.../streams/client/windows/WindowFromFileTest.java | 158 ++++++++
.../streams/client/windows/WindowFromMetaq.java | 47 +++
.../client/windows/WindowHighAvailabilityTest.java | 131 ++++++
.../src/test/resources/log4j.xml | 36 ++
rocketmq-streams-dim/pom.xml | 47 +++
.../apache/rocketmq/streams/dim/DimComponent.java | 63 +++
.../rocketmq/streams/dim/builder/DimBuilder.java | 94 +++++
.../function/expression/InExpressionResource.java | 80 ++++
.../expression/NotInExpressionResource.java | 45 +++
.../dim/function/script/IntelligenceFunction.java | 81 ++++
.../script/IntelligenceNameListFunction.java | 24 ++
.../dim/function/script/NameListFunction.java | 203 ++++++++++
.../rocketmq/streams/dim/index/DimIndex.java | 319 +++++++++++++++
.../rocketmq/streams/dim/index/IndexExecutor.java | 258 ++++++++++++
.../intelligence/AbstractIntelligenceCache.java | 395 +++++++++++++++++++
.../dim/intelligence/AccountIntelligenceCache.java | 77 ++++
.../dim/intelligence/DomainIntelligenceCache.java | 83 ++++
.../dim/intelligence/IPIntelligenceCache.java | 108 +++++
.../dim/intelligence/URLIntelligenceCache.java | 80 ++++
.../rocketmq/streams/dim/model/AbstractDim.java | 312 +++++++++++++++
.../streams/dim/model/BooleanFieldDBDim.java | 55 +++
.../apache/rocketmq/streams/dim/model/DBDim.java | 140 +++++++
.../rocketmq/streams/dim/service/IDimService.java | 65 +++
.../streams/dim/service/impl/DimServiceImpl.java | 92 +++++
.../com/aliyun/service/ConfigureLoaderTest.java | 37 ++
.../com/aliyun/service/ExpressionExecutorTest.java | 80 ++++
.../java/com/aliyun/service/JsonParserTest.java | 40 ++
.../com/aliyun/service/NameListFunctionTest.java | 90 +++++
.../java/com/aliyun/service/TableCompressTest.java | 26 ++
rocketmq-streams-lease/pom.xml | 25 ++
.../rocketmq/streams/lease/LeaseComponent.java | 103 +++++
.../rocketmq/streams/lease/model/LeaseInfo.java | 127 ++++++
.../streams/lease/service/ILeaseGetCallback.java | 30 ++
.../streams/lease/service/ILeaseService.java | 136 +++++++
.../streams/lease/service/ILeaseStorage.java | 73 ++++
.../streams/lease/service/ILeaseStorasge.java | 63 +++
.../lease/service/impl/BasedLesaseImpl.java | 404 +++++++++++++++++++
.../lease/service/impl/LeaseServiceImpl.java | 275 +++++++++++++
.../streams/lease/service/impl/MockLeaseImpl.java | 95 +++++
.../lease/service/storages/DBLeaseStorage.java | 229 +++++++++++
.../rocketmq/streams/lease/LeaseComponentTest.java | 119 ++++++
.../src/test/resources/log4j.xml | 20 +
71 files changed, 8068 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1471653
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,25 @@
+target/
+.DS_Store
+!.mvn/wrapper/maven-wrapper.jar
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+nbproject/private/
+build/
+nbbuild/
+dist/
+nbdist/
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5f3cde5
--- /dev/null
+++ b/README.md
@@ -0,0 +1,108 @@
+# Rocketmq Streams
+
+## Features
+
+* 轻量级部署：可以单独部署，也支持集群部署
+* 多种类型的数据输入以及输出，source支持 rocketmq ， sink支持db, rocketmq 等
+
+## DataStream Example
+
+```java
+import org.apache.rocketmq.streams.client.transform.DataStream;
+
+DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+
+ source
+ .fromFile("/Users/junjie.cheng/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
+```
+
+## Maven Repository
+
+```xml
+
+<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+</dependency>
+```
+
+# Core API
+
+rocketmq-stream 实现了一系列高级的API，可以让用户很方便的编写流计算的程序，实现自己的业务需求；
+
+## StreamBuilder
+
+StreamBuilder 用于构建流任务的源； 内部包含```dataStream()```和```tableStream()```俩个方法，分别返回DataStreamSource和TableStreamSource俩个源；
+
++ [dataStream(nameSpaceName,pipelineName)]() 返回DataStreamSource实例，用于分段编程实现流计算任务；
+
+## DataStream API
+
+### Source
+
+DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据；
+
++ ```fromFile``` 从文件中读取数据， 该方法包含俩个参数
+ + ```filePath``` 文件路径，必填参数
+ + ```isJsonData``` 是否json数据， 非必填参数， 默认为```true```
+
+
++ ```fromRocketmq``` 从rocketmq中获取数据，包含四个参数
+ + ```topic``` rocketmq消息队列的topic名称，必填参数
+ + ```groupName``` 消费者组的名称，必填参数
+ + ```isJson``` 是否json格式，非必填参数
+ + ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
+
+
++ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
+
+### transform
+
+transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
+
+#### DataStream
+
+DataStream实现了一系列常见的流计算算子
+
++ ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream
++ ```flatmap``` 与map类似，一个输入项对应0个或者多个输出项
++ ```filter``` 只选择func返回true的源DStream的记录来返回一个新的DStream
++ ```forEach``` 对每个记录执行一次函数func， 返回一个新的DataStream
++ ```selectFields``` 对每个记录返回对应的字段值，返回一个新的DataStream
++ ```operate``` 对每个记录执行一次自定义的函数，返回一个新的DataStream
++ ```script``` 针对每个记录的字段执行一段脚本，返回新的字段，生成一个新的DataStream
++ ```toPrint``` 将结果在控制台打印，生成新的DataStreamAction实例
++ ```toFile``` 将结果保存为文件，生成一个新的DataStreamAction实例
++ ```toDB``` 将结果保存到数据库
++ ```toRocketmq``` 将结果输出到rocketmq
++ ```toSls``` 将结果输出到sls
++ ```to``` 将结果经过自定义的ISink接口输出到指定的存储
++ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
+ + ```count``` 在窗口内计数
+ + ```min``` 获取窗口内统计值的最小值
+ + ```max``` 获取窗口内统计值得最大值
+ + ```avg``` 获取窗口内统计值的平均值
+ + ```sum``` 获取窗口内统计值的加和值
+ + ```reduce``` 在窗口内进行自定义的汇总运算
++ ```join``` 根据条件将将俩个流进行关联， 合并为一个大流进行相关的运算
++ ```union``` 将俩个流进行合并
++ ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算
++ ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等
+
+# Strategy
+
+策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
+
+```java
+//指定checkpoint的存储策略
+source
+ .fromRocketmq("TSG_META_INFO","")
+ .map(message->message+"--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
+ .start();
+```
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..3675792
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,410 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <name>ROCKETMQ STREAMS</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>rocketmq-streams-commons</module>
+ <module>rocketmq-streams-dim</module>
+ <module>rocketmq-streams-transport-minio</module>
+ <module>rocketmq-streams-script</module>
+ <module>rocketmq-streams-script-python</module>
+ <module>rocketmq-streams-configurable</module>
+ <module>rocketmq-streams-serviceloader</module>
+ <module>rocketmq-streams-filter</module>
+ <module>rocketmq-streams-schedule</module>
+ <module>rocketmq-streams-lease</module>
+ <module>rocketmq-streams-db-operator</module>
+ <module>rocketmq-streams-window</module>
+ <module>rocketmq-streams-clients</module>
+ <module>rocketmq-streams-channel-rocketmq</module>
+ <module>rocketmq-streams-channel-db</module>
+ <module>rocketmq-streams-channel-http</module>
+ </modules>
+
+ <properties>
+ <java.version>1.8</java.version>
+ <java.encoding>UTF-8</java.encoding>
+ <project.build.sourceEncoding>${java.encoding}</project.build.sourceEncoding>
+ <log4j.version>1.2.17</log4j.version>
+ <commons-logging.version>1.1</commons-logging.version>
+ <spring.version>3.2.13.RELEASE</spring.version>
+ <auto-service.version>1.0-rc5</auto-service.version>
+ <mysql-connector.version>5.1.40</mysql-connector.version>
+ <fastjson.version>1.2.27</fastjson.version>
+ <quartz.version>2.2.1</quartz.version>
+ <httpclient.version>4.5.2</httpclient.version>
+ <commons-io.version>2.5</commons-io.version>
+ <junit.version>4.12</junit.version>
+ <guava.version>25.1-jre</guava.version>
+ <groovy.version>2.1.8</groovy.version>
+ <disruptor.version>3.2.0</disruptor.version>
+ <rocksdbjni.version>6.6.4</rocksdbjni.version>
+ <rocketmq.version>4.5.2</rocketmq.version>
+ <hyperscan.version>5.4.0-2.0.0</hyperscan.version>
+ <platform.version>3.5.2</platform.version>
+ <gson.version>2.8.5</gson.version>
+ <java-grok.version>0.1.9</java-grok.version>
+ <jython.version>2.7.0</jython.version>
+ <scala-library.version>2.12.4</scala-library.version>
+ <logback-core.version>1.2.2</logback-core.version>
+ <minio.version>3.0.10</minio.version>
+ </properties>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.18.1</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.5.1</version>
+ <configuration>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
+ <encoding>UTF-8</encoding>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>versions-maven-plugin</artifactId>
+ <version>2.2</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- ================================================= -->
+ <!-- rocketmq streams library -->
+ <!-- ================================================= -->
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-configurable</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-db-operator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-dim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-filter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-lease</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-schedule</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-script</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-script-python</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-serviceloader</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-transport-minio</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-window</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-rocketmq</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- ================================================= -->
+ <!-- rocketmq library -->
+ <!-- ================================================= -->
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-tools</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-common</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-acl</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+
+ <!-- ================================================= -->
+ <!-- tool library -->
+ <!-- ================================================= -->
+
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>${fastjson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons-logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <version>${auto-service.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>${disruptor.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.gliwka.hyperscan</groupId>
+ <artifactId>hyperscan</artifactId>
+ <version>${hyperscan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>platform</artifactId>
+ <version>${platform.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql-connector.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz-jobs</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.krakens</groupId>
+ <artifactId>java-grok</artifactId>
+ <version>${java-grok.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <version>${groovy.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython-standalone</artifactId>
+ <version>${jython.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala-library.version}</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>${logback-core.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio</artifactId>
+ <version>${minio.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ <version>${rocksdbjni.version}</version>
+ </dependency>
+
+ </dependencies>
+ </dependencyManagement>
+
+</project>
diff --git a/rocketmq-streams-clients/pom.xml b/rocketmq-streams-clients/pom.xml
new file mode 100644
index 0000000..ffc051a
--- /dev/null
+++ b/rocketmq-streams-clients/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>rocketmq-streams</artifactId>
+ <groupId>org.apache.rocketmq</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <name>ROCKETMQ STREAMS :: clients</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-commons</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-rocketmq</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-script</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-filter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-window</artifactId>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+
+</project>
\ No newline at end of file
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/DataStreamAction.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/DataStreamAction.java
new file mode 100644
index 0000000..105bd8e
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/DataStreamAction.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.google.common.collect.Maps;
+import org.apache.rocketmq.streams.client.strategy.Strategy;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.topology.ChainPipeline;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+public class DataStreamAction extends DataStream {
+
+ private final Map<String, Object> properties = Maps.newHashMap();
+
+ public DataStreamAction(String namespace, String pipelineName) {
+ super(namespace, pipelineName);
+ }
+
+ public DataStreamAction(PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ super(pipelineBuilder, pipelineBuilders, currentChainStage);
+ }
+
+ public DataStreamAction with(Strategy... strategies) {
+ Properties properties = new Properties();
+ for (Strategy strategy : strategies) {
+ properties.putAll(strategy.getStrategyProperties());
+ }
+ ComponentCreator.createProperties(properties);
+ return this;
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void start() {
+ start(false);
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void asyncStart() {
+ start(true);
+ }
+
+ protected void start(boolean isAsync) {
+ if (this.mainPipelineBuilder == null) {
+ return;
+ }
+ properties.put(ConfigureFileKey.CONNECT_TYPE, "memory");
+ String[] kvs = new String[properties.size()];
+ int i = 0;
+ for (Map.Entry<String, Object> entry : properties.entrySet()) {
+ kvs[i++] = entry.getKey() + ":" + entry.getValue();
+ }
+
+ ConfigurableComponent configurableComponent = ComponentCreator.getComponent(mainPipelineBuilder.getPipelineNameSpace(), ConfigurableComponent.class, kvs);
+ ChainPipeline pipeline = this.mainPipelineBuilder.build(configurableComponent.getService());
+ pipeline.startChannel();
+ if (this.otherPipelineBuilders != null) {
+ for (PipelineBuilder builder : otherPipelineBuilders) {
+ ChainPipeline otherPipeline = builder.build(configurableComponent.getService());
+ otherPipeline.startChannel();
+ }
+ }
+ if (isAsync) {
+ return;
+ }
+ while (true) {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java
new file mode 100644
index 0000000..f67ee2c
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import org.apache.rocketmq.streams.client.source.DataStreamSource;
+
+public class StreamBuilder {
+
+ public static DataStreamSource dataStream(String nameSpaceName, String pipelineName) {
+ return DataStreamSource.create(nameSpaceName, pipelineName);
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
new file mode 100644
index 0000000..8d71c31
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.source;
+
+import com.google.common.collect.Sets;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.channel.impl.file.FileSource;
+import org.apache.rocketmq.streams.common.channel.source.ISource;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.source.RocketMQSource;
+
+import java.util.Set;
+
+public class DataStreamSource {
+ protected PipelineBuilder mainPipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+
+ public DataStreamSource(String namespace, String pipelineName) {
+ this.mainPipelineBuilder = new PipelineBuilder(namespace, pipelineName);
+ this.otherPipelineBuilders = Sets.newHashSet();
+ }
+
+ public static DataStreamSource create(String namespace, String pipelineName) {
+ return new DataStreamSource(namespace, pipelineName);
+ }
+
+ public DataStream fromFile(String filePath) {
+ return fromFile(filePath, true);
+ }
+
+ public DataStream fromFile(String filePath, Boolean isJsonData) {
+ FileSource fileChannel = new FileSource(filePath);
+ fileChannel.setJsonData(isJsonData);
+ this.mainPipelineBuilder.setSource(fileChannel);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
+ }
+
+ public DataStream fromRocketmq(String topic, String groupName) {
+ return fromRocketmq(topic, groupName, null, false);
+ }
+
+ public DataStream fromRocketmq(String topic, String groupName, boolean isJson) {
+ return fromRocketmq(topic, groupName, null, isJson);
+ }
+
+ public DataStream fromRocketmq(String topic, String groupName, String tags, boolean isJson) {
+ RocketMQSource rocketMQSource = new RocketMQSource();
+ rocketMQSource.setTopic(topic);
+ rocketMQSource.setTags(tags);
+ rocketMQSource.setGroupName(groupName);
+ rocketMQSource.setJsonData(isJson);
+ this.mainPipelineBuilder.setSource(rocketMQSource);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
+ }
+
+ public DataStream from(ISource<?> source) {
+ this.mainPipelineBuilder.setSource(source);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/CheckpointStrategy.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/CheckpointStrategy.java
new file mode 100644
index 0000000..20494c4
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/CheckpointStrategy.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.strategy;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+
+import java.util.Properties;
+
+public class CheckpointStrategy implements Strategy {
+
+ private final Properties properties;
+
+ private CheckpointStrategy(Long pollingTime) {
+ properties = new Properties();
+ properties.put(AbstractComponent.CONNECT_TYPE, IConfigurableService.MEMORY_SERVICE_NAME);
+ properties.put(AbstractComponent.POLLING_TIME, pollingTime + "");
+ }
+
+ private CheckpointStrategy(String filePath, Long pollingTime) {
+ properties = new Properties();
+ properties.put(AbstractComponent.CONNECT_TYPE, IConfigurableService.FILE_SERVICE_NAME);
+ properties.put(IConfigurableService.FILE_PATH_NAME, filePath);
+ properties.put(AbstractComponent.POLLING_TIME, pollingTime + "");
+ }
+
+ private CheckpointStrategy(String url, String username, String password, Long pollingTime) {
+ properties = new Properties();
+ properties.put(AbstractComponent.JDBC_DRIVER, AbstractComponent.DEFAULT_JDBC_DRIVER);
+ properties.put(AbstractComponent.JDBC_URL, url);
+ properties.put(AbstractComponent.JDBC_USERNAME, username);
+ properties.put(AbstractComponent.JDBC_PASSWORD, password);
+ properties.put(AbstractComponent.JDBC_TABLE_NAME, AbstractComponent.DEFAULT_JDBC_TABLE_NAME);
+ properties.put(AbstractComponent.POLLING_TIME, pollingTime + "");
+ properties.put(AbstractComponent.CONNECT_TYPE, IConfigurableService.DEFAULT_SERVICE_NAME);
+ }
+
+ @Override
+ public Properties getStrategyProperties() {
+ return this.properties;
+ }
+
+ public static Strategy db(String url, String username, String password, Long pollingTime) {
+ return new CheckpointStrategy(url, username, password, pollingTime);
+ }
+
+ public static Strategy file(String filePath, Long pollingTime) {
+ return new CheckpointStrategy(filePath, pollingTime);
+ }
+
+ public static Strategy mem(Long pollingTime) {
+ return new CheckpointStrategy(pollingTime);
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java
new file mode 100644
index 0000000..c647759
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.strategy;
+
+import java.util.Properties;
+
+public class StateStrategy implements Strategy {
+
+ private Properties properties;
+
+ private StateStrategy() {
+ }
+
+ @Override
+ public Properties getStrategyProperties() {
+ return this.properties;
+ }
+
+ public static Strategy db() {
+ return new StateStrategy();
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/Strategy.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/Strategy.java
new file mode 100644
index 0000000..727c645
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/Strategy.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.strategy;
+
+import java.util.Properties;
+
+public interface Strategy {
+
+ Properties getStrategyProperties();
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
new file mode 100644
index 0000000..41aae96
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
@@ -0,0 +1,437 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Sets;
+import org.apache.rocketmq.streams.client.DataStreamAction;
+import org.apache.rocketmq.streams.client.transform.window.WindowInfo;
+import org.apache.rocketmq.streams.common.channel.impl.OutputPrintChannel;
+import org.apache.rocketmq.streams.common.channel.impl.file.FileSink;
+import org.apache.rocketmq.streams.common.channel.sink.ISink;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.context.AbstractContext;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.context.MessageHeader;
+import org.apache.rocketmq.streams.common.context.UserDefinedMessage;
+import org.apache.rocketmq.streams.common.functions.*;
+import org.apache.rocketmq.streams.common.topology.ChainPipeline;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.IStageBuilder;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.topology.model.Union;
+import org.apache.rocketmq.streams.common.topology.stages.udf.StageBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.udf.UDFUnionChainStage;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.db.sink.DBSink;
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.apache.rocketmq.streams.filter.operator.FilterOperator;
+import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator;
+import org.apache.rocketmq.streams.sink.RocketMQSink;
+import org.apache.rocketmq.streams.window.builder.WindowBuilder;
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+import org.apache.rocketmq.streams.window.operator.join.JoinWindow;
+
+import java.io.Serializable;
+import java.util.Set;
+
+public class DataStream implements Serializable {
+
+ protected PipelineBuilder mainPipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+ public DataStream(String namespace, String pipelineName) {
+ this.mainPipelineBuilder = new PipelineBuilder(namespace, pipelineName);
+ this.otherPipelineBuilders = Sets.newHashSet();
+ }
+
+ public DataStream(PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.mainPipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ }
+
+ public DataStream script(String script) {
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(new ScriptOperator(script));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ public DataStream filter(String expressions) {
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(new FilterOperator(expressions));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ public <T, O> DataStream map(MapFunction<T, O> mapFunction) {
+ StageBuilder stageBuilder = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ try {
+ O o = (O)(message.getMessageValue());
+ T result = (T)mapFunction.map(o);
+ if (result != message.getMessageValue()) {
+ if (result instanceof JSONObject) {
+ message.setMessageBody((JSONObject)result);
+ } else {
+ message.setMessageBody(new UserDefinedMessage(result));
+ }
+ }
+ return null;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ };
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(stageBuilder);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ public <O> DataStream filter(final FilterFunction<O> filterFunction) {
+ StageBuilder mapUDFOperator = new StageBuilder() {
+
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ try {
+ boolean isFilter = filterFunction.filter((O)message.getMessageValue());
+ if (isFilter) {
+ context.breakExecute();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ };
+ ChainStage stage = this.mainPipelineBuilder.createStage(mapUDFOperator);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * windows streams
+ *
+ * @param windowInfo 通过不同窗口类型动of方法创建，SessionWindow.of(Time.seconds(10))
+ * @return WindowStream
+ */
+ public WindowStream window(WindowInfo windowInfo) {
+ AbstractWindow window = windowInfo.createWindow();
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(window);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new WindowStream(window, this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 通用增加stage的方法，低级接口，适合用户自定义stage的场景
+ *
+ * @param stageBuilder 创建stage和变量的接口
+ * @return DataStream
+ */
+ public DataStream addStage(IStageBuilder stageBuilder) {
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(stageBuilder);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 创建join stream。实现原理，通过共享一个JoinWindow，并打标左右流，在join windown完成缓存，join逻辑
+ *
+ * @param rightStream 通过不同窗口类型动of方法创建，SessionWindow.of(Time.seconds(10))
+ * @return
+ */
+ public JoinStream join(DataStream rightStream) {
+ JoinWindow window = WindowBuilder.createDefaultJoinWindow();
+ //处理左边分支，增加map，主要是增加msg msgRouteFromLable->增加窗口stage
+ ChainStage<?> leftScriptStage = this.mainPipelineBuilder.createStage(new ScriptOperator("setHeader(msgRouteFromLable,'" + MessageHeader.JOIN_LEFT + "')"));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, leftScriptStage);
+ this.currentChainStage = leftScriptStage;
+ ChainStage<?> leftWindowStage = this.mainPipelineBuilder.createStage(window);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, leftWindowStage);
+
+ //处理右流，右流增加script
+ DataStream dataStream = rightStream.script("setHeader(msgRouteFromLable,'" + MessageHeader.JOIN_RIGHT + "')").addStage(window);
+ //dataStream.addStage(window);
+
+ addOtherDataStream(rightStream);
+ return new JoinStream(window, this.mainPipelineBuilder, this.otherPipelineBuilders, leftWindowStage);
+ }
+
+ /**
+ * 通过共享对象union，完成两个数据汇聚，左流需要设置isMainStream=true
+ *
+ * @param rightStream
+ * @return
+ */
+ public DataStream union(DataStream rightStream) {
+ Union union = new Union();
+
+ //处理左流，做流的isMain设置成true
+ UDFUnionChainStage chainStage = (UDFUnionChainStage)this.mainPipelineBuilder.createStage(union);
+ chainStage.setMainStream(true);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, chainStage);
+
+ //处理右流，做流的isMain设置成true
+ rightStream.addStage(union);
+
+ addOtherDataStream(rightStream);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, chainStage);
+ }
+
+ /**
+ * 把一个流拆分成多个流，通过设置不同流的标签实现
+ *
+ * @param splitFunction 拆分流的具体逻辑
+ * @return
+ */
+ public SplitStream split(SplitFunction splitFunction) {
+ StageBuilder operator = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ String labelName = splitFunction.split(message.getMessageValue());
+ message.getHeader().addRouteLable(labelName);
+ return null;
+ }
+ };
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(operator);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new SplitStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 维表join,mysql场景，不需要指定jdbcdriver
+ *
+ * @param url
+ * @param userName
+ * @param password
+ * @param sqlOrTableName
+ * @return
+ */
+ public JoinStream join(String url, String userName, String password, String sqlOrTableName, long pollingTimeMintue) {
+ return join(url, userName, password, sqlOrTableName, null, pollingTimeMintue);
+ }
+
+ /**
+ * 维表join
+ *
+ * @param url
+ * @param userName
+ * @param password
+ * @param sqlOrTableName
+ * @return
+ */
+ public JoinStream join(String url, String userName, String password, String sqlOrTableName, String jdbcDriver, long pollingTimeMinute) {
+ DBDim dbDim = new DBDim();
+ dbDim.setUrl(url);
+ dbDim.setUserName(userName);
+ dbDim.setPassword(password);
+ dbDim.setSql(sqlOrTableName);
+ dbDim.setPollingTimeMintue(pollingTimeMinute);
+ dbDim.setJdbcdriver(jdbcDriver);
+ this.mainPipelineBuilder.addConfigurables(dbDim);
+ return new JoinStream(dbDim, mainPipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+
+ /**
+ * 遍历所有数据
+ *
+ * @param forEachFunction
+ * @param <O>
+ * @return
+ */
+ public <O> DataStream forEach(ForEachFunction<O> forEachFunction) {
+ StageBuilder selfChainStage = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ forEachFunction.foreach((O)message.getMessageValue());
+ return null;
+ }
+ };
+ ChainStage stage = this.mainPipelineBuilder.createStage(selfChainStage);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 遍历所有数据
+ *
+ * @param forEachFunction
+ * @param <O>
+ * @return
+ */
+ public <O> DataStream forEachMessage(ForEachMessageFunction forEachFunction) {
+ StageBuilder selfChainStage = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ forEachFunction.foreach(message, context);
+ return null;
+ }
+ };
+ ChainStage stage = this.mainPipelineBuilder.createStage(selfChainStage);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 只保留需要的字段
+ *
+ * @param fieldNames
+ */
+ public DataStream selectFields(String... fieldNames) {
+ ChainStage stage = this.mainPipelineBuilder.createStage(new ScriptOperator("retain(" + MapKeyUtil.createKeyBySign(",", fieldNames) + ")"));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void start() {
+ start(false);
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void asynStart() {
+ start(true);
+ }
+
+ protected void start(boolean isAsyn) {
+ if (this.mainPipelineBuilder == null) {
+ return;
+ }
+ ConfigurableComponent configurableComponent = ComponentCreator.getComponent(mainPipelineBuilder.getPipelineNameSpace(), ConfigurableComponent.class, ConfigureFileKey.CONNECT_TYPE + ":memory");
+ ChainPipeline pipeline = this.mainPipelineBuilder.build(configurableComponent.getService());
+ pipeline.startChannel();
+ if (this.otherPipelineBuilders != null) {
+ for (PipelineBuilder builder : otherPipelineBuilders) {
+ ChainPipeline otherPipeline = builder.build(configurableComponent.getService());
+ otherPipeline.startChannel();
+ }
+ }
+ if (isAsyn) {
+ return;
+ }
+ while (true) {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * 把其他流的 pipelinebuilder 放到set中
+ *
+ * @param rightSource
+ */
+ protected void addOtherDataStream(DataStream rightSource) {
+ //如果是多流join，需要把把piplinebuider保存下来，在启动时，启动多个pipline
+ if (!rightSource.mainPipelineBuilder.equals(this.mainPipelineBuilder)) {
+ this.otherPipelineBuilders.add(rightSource.mainPipelineBuilder);
+ }
+
+ this.otherPipelineBuilders.addAll(rightSource.otherPipelineBuilders);
+ }
+
+ public DataStreamAction toFile(String filePath) {
+ FileSink fileChannel = new FileSink(filePath);
+ ChainStage<?> output = mainPipelineBuilder.createStage(fileChannel);
+ mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toPrint() {
+ return toPrint(-1);
+ }
+
+ public DataStreamAction toPrint(int batchSize) {
+ OutputPrintChannel outputPrintChannel = new OutputPrintChannel();
+ if (batchSize > 0) {
+ outputPrintChannel.setBatchSize(batchSize);
+ }
+ ChainStage output = this.mainPipelineBuilder.createStage(outputPrintChannel);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toDB(String url, String userName, String password, String tableName) {
+ DBSink dbChannel = new DBSink(url, userName, password);
+ dbChannel.setTableName(tableName);
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(dbChannel);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toMetaq(String topic) {
+ return toMetaq(topic, null);
+ }
+
+ public DataStreamAction toMetaq(String topic, String tags) {
+ return toMetaq(topic, tags, -1);
+ }
+
+ public DataStreamAction toMetaq(String topic, String tags, int batchSize) {
+ RocketMQSink metaqChannel = new RocketMQSink();
+ metaqChannel.setTopic(topic);
+ metaqChannel.setTags(tags);
+ if (batchSize > 0) {
+ metaqChannel.setBatchSize(batchSize);
+ }
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(metaqChannel);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toRocketmq(String topic, String groupName, String endpoint, String namesrvAddr,
+ String accessKey, String secretKey, String instanceId) {
+ return toRocketmq(topic, "*", groupName, namesrvAddr, endpoint, accessKey, secretKey, instanceId);
+ }
+
+ public DataStreamAction toRocketmq(String topic, String tags, String groupName, String endpoint,
+ String namesrvAddr, String accessKey, String secretKey, String instanceId) {
+ return toRocketmq(topic, tags, -1, groupName, namesrvAddr, endpoint, accessKey, secretKey, instanceId);
+ }
+
+ public DataStreamAction toRocketmq(String topic, String tags, int batchSize, String groupName,
+ String endpoint, String namesrvAddr, String accessKey, String secretKey, String instanceId) {
+ RocketMQSink rocketMQSink = new RocketMQSink();
+ rocketMQSink.setTopic(topic);
+ rocketMQSink.setTags(tags);
+ rocketMQSink.setGroupName(groupName);
+ rocketMQSink.setNamesrvAddr(namesrvAddr);
+ if (batchSize > 0) {
+ rocketMQSink.setBatchSize(batchSize);
+ }
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(rocketMQSink);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction to(ISink<?> sink) {
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(sink);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/JoinStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/JoinStream.java
new file mode 100644
index 0000000..26e6420
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/JoinStream.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.common.model.NameCreator;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.function.expression.Equals;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression;
+import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator;
+import org.apache.rocketmq.streams.window.operator.join.JoinWindow;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class JoinStream {
+
+ private static final String INNER_VAR_NAME_PREFIX = "___";
+ protected JoinWindow joinWindow;//完成join 条件的添加
+ protected boolean isDimJoin = false;//是否是维表join
+
+ protected AbstractDim dim;//维度表对象
+ protected String onCondition;//条件
+ protected JoinType joinType;//连接类型
+
+ //用于返回DataStream流
+ protected PipelineBuilder pipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+ /**
+ * 双流join 场景
+ *
+ * @param joinWindow
+ * @param pipelineBuilder
+ * @param pipelineBuilders
+ * @param currentChainStage
+ */
+ public JoinStream(JoinWindow joinWindow, PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ this.joinWindow = joinWindow;
+ }
+
+ /**
+ * 维表join 场景
+ *
+ * @param pipelineBuilder
+ * @param pipelineBuilders
+ * @param currentChainStage
+ */
+ public JoinStream(AbstractDim dim, PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ this.dim = dim;
+ }
+
+ public JoinStream setJoinType(JoinType joinType) {
+ this.joinType = joinType;
+
+ return this;
+ }
+
+ /**
+ * 指定窗口，如果不指定，默认1个小时
+ *
+ * @param time
+ * @return
+ */
+ public JoinStream window(Time time) {
+
+ //维表join 不需要设置
+ if (isDimJoin) {
+ throw new RuntimeException("can not support this method");
+ }
+ joinWindow.setTimeUnitAdjust(1);
+ joinWindow.setSizeInterval(time.getValue());
+ joinWindow.setSlideInterval(time.getValue());
+ joinWindow.setRetainWindowCount(1);
+ return this;
+ }
+
+ /**
+ * 增加条件,用表达式形式表达(leftFieldName,function,rightFieldName)&&({name,==,otherName}||(age,==,age)) 后续再增加结构化的方法
+ *
+ * @param onCondition (leftFieldName,function,rightFieldName)&&({name,==,otherName}||(age,==,age))
+ * @return
+ */
+ public JoinStream setCondition(String onCondition) {
+ this.onCondition = onCondition;
+ return this;
+ }
+
+ public DataStream toDataSteam() {
+ if (isDimJoin) {
+ return doDimJoin();
+ } else {
+ return doJoin();
+ }
+
+ }
+
+ /**
+ * 维度表join的场景
+ */
+ protected DataStream doDimJoin() {
+ String script = null;
+ if (JoinType.INNER_JOIN == joinType) {
+ String data = createName("inner_join");
+ script = data + "=inner_join('" + dim.getNameSpace() + "','" + dim.getConfigureName() + "','" + onCondition + "'," + null + ",''," + null + ");splitArray('" + data + "');";
+ } else if ((JoinType.LEFT_JOIN == joinType)) {
+ String data = createName("left_join");
+ script = data + "=left_join('" + dim.getNameSpace() + "','" + dim.getConfigureName() + "','" + onCondition + "'," + null + ",'" + null + "'," + null + ");if(!null(" + data + ")){splitArray('" + data + "');};";
+ }
+ ChainStage stage = this.pipelineBuilder.createStage(new ScriptOperator(script));
+ this.pipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 双流join的场景
+ */
+ protected DataStream doJoin() {
+ if (JoinType.INNER_JOIN == joinType) {
+ joinWindow.setJoinType("INNER");
+ } else if (JoinType.LEFT_JOIN == joinType) {
+ joinWindow.setJoinType("LEFT");
+ } else {
+ throw new RuntimeException("can not support this join type, expect INNER,LEFT, real is " + joinType.toString());
+ }
+
+ AtomicBoolean hasNoEqualsExpression = new AtomicBoolean(false);//是否有非等值的join 条件
+ Map<String, String> left2Right = createJoinFieldsFromCondition(onCondition, hasNoEqualsExpression);//把等值条件的左右字段映射成map
+ List<String> leftList = new ArrayList<>();
+ List<String> rightList = new ArrayList<>();
+ leftList.addAll(left2Right.keySet());
+ rightList.addAll(left2Right.values());
+ joinWindow.setLeftJoinFieldNames(leftList);
+ joinWindow.setRightJoinFieldNames(rightList);
+ //如果有非等值，则把这个条件设置进去
+ if (hasNoEqualsExpression.get()) {
+ joinWindow.setExpression(onCondition);
+ }
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+
+ /**
+ * 支持的连接类型，目前支持inner join和left join
+ */
+ public enum JoinType {
+ INNER_JOIN,
+ LEFT_JOIN
+ }
+
+ /**
+ * 从条件中找到join 左右的字段。如果有非等值，则不包含在内
+ *
+ * @param
+ * @param onCondition
+ * @return
+ */
+ public Map<String, String> createJoinFieldsFromCondition(String onCondition, AtomicBoolean hasNoEqualsExpression) {
+ List<Expression> expressions = new ArrayList<>();
+ List<RelationExpression> relationExpressions = new ArrayList<>();
+ ExpressionBuilder.createOptimizationExpression("tmp", "tmp", onCondition, expressions, relationExpressions);
+ Map<String, String> left2Right = new HashMap<>();
+ for (Expression expression : expressions) {
+ String varName = expression.getVarName();
+ String valueName = expression.getValue().toString();
+ if (!Equals.isEqualFunction(expression.getFunctionName())) {
+ hasNoEqualsExpression.set(true);
+ continue;
+ }
+ left2Right.put(varName, valueName);
+ }
+ return left2Right;
+ }
+
+ public static String createName(String functionName, String... names) {
+ if (names == null || names.length == 0) {
+ return NameCreator.createNewName(INNER_VAR_NAME_PREFIX, functionName);
+ }
+ String[] values = new String[names.length + 2];
+ values[0] = INNER_VAR_NAME_PREFIX;
+ values[1] = functionName;
+ for (int i = 2; i < values.length; i++) {
+ values[i] = names[i - 2];
+ }
+ return NameCreator.createNewName(values);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/SplitStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/SplitStream.java
new file mode 100644
index 0000000..49efcd2
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/SplitStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import org.apache.rocketmq.streams.common.context.AbstractContext;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.udf.StageBuilder;
+
+import java.util.Set;
+
+public class SplitStream {
+
+
+ /**
+ * 创建datastream时使用
+ */
+ protected PipelineBuilder pipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+
+ public SplitStream(PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ }
+
+ /**
+ * 选择一个分支
+ * @param lableName
+ * @return
+ */
+ public DataStream select(String lableName){
+ StageBuilder stage = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ return null;
+ }
+ };
+ stage.setLabel(lableName);
+ this.pipelineBuilder.setTopologyStages(currentChainStage,stage);
+ return new DataStream(pipelineBuilder,otherPipelineBuilders,stage);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/WindowStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/WindowStream.java
new file mode 100644
index 0000000..1f6ea24
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/WindowStream.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.rocketmq.streams.common.context.UserDefinedMessage;
+import org.apache.rocketmq.streams.common.functions.ReduceFunction;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.udf.IReducer;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+
+import java.util.Set;
+
+/**
+ * 做windown 相关操作 可以同时设置多个统计算子，如count，sum，avg 通过toDataSteam/reduce 返回DataSteam
+ */
+public class WindowStream {
+ //window 对象
+ protected AbstractWindow window;
+
+ /**
+ * 创建datastream时使用
+ */
+ protected PipelineBuilder pipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+ public WindowStream(AbstractWindow window, PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ this.window = window;
+ }
+
+ /**
+ * 做count算子
+ *
+ * @param asName count结果对应的名字，如 sql中count(1) as c 。asName=c
+ * @return
+ */
+ public WindowStream count(String asName) {
+ window.getSelectMap().put(asName, asName + "=count(" + asName + ")");
+ return this;
+ }
+
+ /**
+ * 做min算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @return
+ */
+ public WindowStream min(String fieldName) {
+ window.getSelectMap().put(fieldName, fieldName + "=min(" + fieldName + ")");
+ return this;
+ }
+
+ /**
+ * 做max算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @return
+ */
+ public WindowStream max(String fieldName) {
+ window.getSelectMap().put(fieldName, fieldName + "=max(" + fieldName + ")");
+ return this;
+ }
+
+ /**
+ * 做avg算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @param asName avg结果对应的名字，如 sql中avg(name) as c 。asName=c
+ * @return
+ */
+ public WindowStream avg(String fieldName, String asName) {
+ window.getSelectMap().put(asName, asName + "=avg(" + fieldName + ")");
+ return this;
+ }
+
+ /**
+ * 做sum算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @param asName sum结果对应的名字，如 sql中sum(name) as c 。asName=c
+ * @return
+ */
+ public WindowStream sum(String fieldName, String asName) {
+ window.getSelectMap().put(asName, asName + "=sum(" + fieldName + ")");
+ return this;
+ }
+
+ public WindowStream setTimeField(String timeField) {
+ window.setTimeFieldName(timeField);
+ return this;
+ }
+
+ public WindowStream setFireMode(int fireMode) {
+ window.setFireMode(fireMode);
+ return this;
+ }
+
+ public WindowStream setLocalStorageOnly(boolean isLocalStorageOnley) {
+ window.setLocalStorageOnly(isLocalStorageOnley);
+ return this;
+ }
+
+ public WindowStream setMaxMsgGap(Long maxMsgGapSecond) {
+ window.setMsgMaxGapSecond(maxMsgGapSecond);
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param fieldNames
+ * @return
+ */
+ public WindowStream groupBy(String... fieldNames) {
+ window.setGroupByFieldName(MapKeyUtil.createKeyBySign(";", fieldNames));
+ for (String fieldName : fieldNames) {
+ window.getSelectMap().put(fieldName, fieldName);
+ }
+
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param fireMode
+ * @return
+ */
+ public WindowStream fireMode(int fireMode) {
+ window.setFireMode(fireMode);
+
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param waterMarkSecond
+ * @return
+ */
+ public WindowStream waterMark(int waterMarkSecond) {
+ window.setWaterMarkMinute(waterMarkSecond);
+
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param fieldName
+ * @return
+ */
+ public WindowStream timeField(String fieldName) {
+ window.setTimeFieldName(fieldName);
+
+ return this;
+ }
+
+ /**
+ * 用户自定义reduce逻辑
+ *
+ * @param reduceFunction
+ * @return
+ */
+ public <R, O> DataStream reduce(ReduceFunction<R, O> reduceFunction) {
+ window.setReducer(new IReducer() {
+ @Override
+ public JSONObject reduce(JSONObject accumulator, JSONObject msg) {
+ Object accumulatorValue = accumulator;
+ Object msgValue = msg;
+ if (msg instanceof UserDefinedMessage) {
+ UserDefinedMessage userDefinedMessage = (UserDefinedMessage)msg;
+ msgValue = userDefinedMessage.getMessageValue();
+ }
+ if (accumulator instanceof UserDefinedMessage) {
+ UserDefinedMessage userDefinedMessage = (UserDefinedMessage)accumulator;
+ accumulatorValue = userDefinedMessage.getMessageValue();
+ }
+ R result = reduceFunction.reduce((R)accumulatorValue, (O)msgValue);
+ return new UserDefinedMessage(result);
+ }
+ });
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+
+ public DataStream toDataSteam() {
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/HoppingWindow.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/HoppingWindow.java
new file mode 100644
index 0000000..0c945f0
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/HoppingWindow.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+public class HoppingWindow {
+ /**
+ * 滑动窗口信息
+ * @return
+ */
+ public static WindowInfo of(Time windowSize,Time windowSlide){
+ WindowInfo windowInfo=new WindowInfo();
+ windowInfo.setType(WindowInfo.HOPPING_WINDOW);
+ windowInfo.setWindowSize(windowSize);
+ windowInfo.setWindowSlide(windowSlide);
+ return windowInfo;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/SessionWindow.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/SessionWindow.java
new file mode 100644
index 0000000..67c7f22
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/SessionWindow.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+public class SessionWindow {
+ /**
+ * 滑动窗口信息
+ * @param time
+ * @return
+ */
+ public static WindowInfo of(Time time){
+ WindowInfo windowInfo=new WindowInfo();
+ windowInfo.setType(WindowInfo.SESSION_WINDOW);
+ windowInfo.setWindowSize(time);
+ return windowInfo;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/Time.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/Time.java
new file mode 100644
index 0000000..2157e01
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/Time.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+/**
+ * 给window指定窗口时间
+ */
+public class Time {
+ protected int value;
+ public Time(int value){
+ this.value=value;
+ }
+ public static Time seconds(int second){
+
+ return new Time(second);
+ }
+ public static Time minutes(int minutes){
+ return new Time(minutes*60);
+ }
+ public static Time hours(int hours){
+ return new Time(60*60*hours);
+ }
+ public static Time days(int days){
+ return new Time(60*60*24*days);
+ }
+
+ public int getValue() {
+ return value;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/TumblingWindow.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/TumblingWindow.java
new file mode 100644
index 0000000..62c0ce3
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/TumblingWindow.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+public class TumblingWindow {
+ /**
+ * 滑动窗口信息
+ *
+ * @param time
+ * @return
+ */
+ public static WindowInfo of(Time time) {
+ WindowInfo windowInfo = new WindowInfo();
+ windowInfo.setType(WindowInfo.TUMBLING_WINDOW);
+ windowInfo.setWindowSize(time);
+ return windowInfo;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
new file mode 100644
index 0000000..64be503
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.transform.window;
+
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+import org.apache.rocketmq.streams.window.operator.impl.SessionWindow;
+import org.apache.rocketmq.streams.window.operator.impl.WindowOperator;
+
+/**
+ * 保存创建window的信息 主要是窗口类型，窗口大小
+ */
+public class WindowInfo {
+ public static int HOPPING_WINDOW = 1;//滑动窗口
+ public static int TUMBLING_WINDOW = 2;//滚动窗口
+ public static int SESSION_WINDOW = 23;
+ protected int type;//window类型 hopping，Tumbling
+ protected Time windowSize;//窗口大小
+ protected Time windowSlide;//滑动大小
+
+ /**
+ * 创建窗口
+ *
+ * @return
+ */
+ public AbstractWindow createWindow() {
+ AbstractWindow window = null;
+ if (type == HOPPING_WINDOW) {
+ window = new WindowOperator();
+ window.setTimeUnitAdjust(1);
+ window.setSizeInterval(windowSize.getValue());
+ window.setSlideInterval(windowSlide.getValue());
+ } else if (type == TUMBLING_WINDOW) {
+ window = new WindowOperator();
+ window.setTimeUnitAdjust(1);
+ window.setSizeInterval(windowSize.getValue());
+ } else if (type == SESSION_WINDOW) {
+ window = new SessionWindow();
+ window.setTimeUnitAdjust(1);
+ window.setSizeInterval(windowSize.getValue());
+ } else {
+ throw new RuntimeException("can not support the type ,expect 1，2，3。actual is " + type);
+ }
+ return window;
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public void setType(int type) {
+ this.type = type;
+ }
+
+ public Time getWindowSize() {
+ return windowSize;
+ }
+
+ public void setWindowSize(Time windowSize) {
+ this.windowSize = windowSize;
+ }
+
+ public Time getWindowSlide() {
+ return windowSlide;
+ }
+
+ public void setWindowSlide(Time windowSlide) {
+ this.windowSlide = windowSlide;
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
new file mode 100644
index 0000000..7e078f8
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.configuable.ConfigurableComponent;
+import org.apache.rocketmq.streams.configuable.model.Configure;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * 数据库的存储，需要配置存储的连接参数，请先完成配置，后执行单元用例 如果未建表，可以通过Configure.createTableSQL() 获取建表语句，创建表后，测试
+ */
+public class DBDriverTest {
+ private String URL = "";
+ protected String USER_NAME = "";
+ protected String PASSWORD = "";
+ protected String TABLE_NAME = "rocketmq_streams_configure_source";
+
+ @Test
+ public void testDBConfigurableService() {
+ String namespace = "streams.db.configurable";
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE, "DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_TABLE_NAME, TABLE_NAME);
+
+ //如果表不存在，创建表
+ String sql = (Configure.createTableSQL(TABLE_NAME));
+ DriverBuilder.createDriver().execute(sql);
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ configurableComponent.insert(createPerson(namespace));
+ configurableComponent.refreshConfigurable(namespace);
+ Person person = configurableComponent.queryConfigurable("person", "peronName");
+ assertNotNull(person);
+ }
+
+ /**
+ * 创建configuable对象
+ *
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setName("chris");
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DataStreamTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DataStreamTest.java
new file mode 100644
index 0000000..8f54067
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DataStreamTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import org.apache.rocketmq.streams.client.source.DataStreamSource;
+import org.apache.rocketmq.streams.client.strategy.CheckpointStrategy;
+import org.apache.rocketmq.streams.client.strategy.StateStrategy;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.sql.*;
+
+public class DataStreamTest implements Serializable {
+
+ DataStreamSource dataStream;
+
+ @Before
+ public void init() {
+ dataStream = StreamBuilder.dataStream("test_namespace", "graph_pipeline");
+ }
+
+ @Test
+ public void testFromFile() {
+ dataStream
+ .fromFile("/Users/junjie.cheng/text.txt", false)
+ .map(message -> message + "--")
+ .toPrint(1)
+ .start();
+ }
+
+ @Test
+ public void testRocketmq() {
+ dataStream
+ .fromRocketmq("TOPIC_EVENT_SAS_SECURITY_EVENT", "111")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .start();
+ }
+
+ @Test
+ public void testDBCheckPoint() {
+ dataStream
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("", "", "", 0L))
+ .start();
+ }
+
+ @Test
+ public void testFileCheckPoint() {
+ dataStream
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.mem(0L))
+ .start();
+ }
+
+ @Test
+ public void testBothStrategy() {
+ dataStream
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.mem(0L), StateStrategy.db())
+ .start();
+ }
+
+ @Test
+ public void testMeta() {
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ Connection connection = DriverManager.getConnection("", "", "");
+ DatabaseMetaData metaData = connection.getMetaData();
+ ResultSet dataFilter = metaData.getColumns(connection.getCatalog(), "%", "XXX", null);
+ while (dataFilter.next()) {
+ String one = dataFilter.getString("DATA_TYPE");
+ int two = dataFilter.getInt("COLUMN_SIZE");
+ String three = dataFilter.getString("COLUMN_NAME");
+ String four = dataFilter.getString("TYPE_NAME");
+ System.out.println(one + " " + two + " " + three + " " + four + " " + DataTypeUtil.getDataType(dataFilter.getString("TYPE_NAME")));
+ }
+
+ } catch (ClassNotFoundException | SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/FilterTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/FilterTest.java
new file mode 100644
index 0000000..19665fd
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/FilterTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.util.List;
+import org.apache.rocketmq.streams.filter.FilterComponent;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+public class FilterTest {
+ @Test
+ public void testFilter(){
+ JSONObject msg=new JSONObject();
+ msg.put("name","chris");
+ msg.put("age",18);
+ Rule rule= ExpressionBuilder.createRule("tmp","tmp","(name,==,chris)&(age,>=,18)");
+ FilterComponent filterComponent=FilterComponent.getInstance();
+ List<Rule> fireRules=filterComponent.excuteRule(msg,rule);
+ assertTrue(fireRules.size()==1);
+ }
+
+ @Test
+ public void testFilter2(){
+ JSONObject msg=new JSONObject();
+ msg.put("name","chris");
+ msg.put("age",18);
+ boolean result= ExpressionBuilder.executeExecute("tmp","(name,==,chris)&(age,>,int,18)",msg);
+ assertTrue(!result);
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/JoinTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/JoinTest.java
new file mode 100644
index 0000000..b221e92
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/JoinTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.JoinStream.JoinType;
+import org.apache.rocketmq.streams.common.functions.FilterFunction;
+import org.junit.Test;
+
+public class JoinTest implements Serializable {
+
+ @Test
+ public void testJoin() {
+ DataStream leftStream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ DataStream rightStream = (StreamBuilder.dataStream("namespace", "name2")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ leftStream.join(rightStream).setJoinType(JoinType.INNER_JOIN)
+ .setCondition("(ProjectName,==,ProjectName)&(LogStore,==,LogStore)")
+ .toDataSteam()
+ .toPrint()
+ .start();
+
+ }
+
+ @Test
+ public void testDim() {
+ DataStreamAction stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }))
+ .join("dburl", "dbUserName", "dbPassowrd", "tableNameOrSQL", 5)
+ .setCondition("(name,==,name)")
+ .toDataSteam()
+ .selectFields("name", "age", "address")
+ .toPrint();
+ stream.start();
+ ;
+ ;
+
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/LeaseTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/LeaseTest.java
new file mode 100644
index 0000000..1d3ec8a
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/LeaseTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import java.util.Date;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.lease.LeaseComponent;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class LeaseTest {
+
+ private String URL="";
+ protected String USER_NAME="";
+ protected String PASSWORD="";
+
+ public LeaseTest(){
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE,"DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL,URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME,USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD,PASSWORD);//password
+
+ /**
+ * 创建lease info表
+ */
+ JDBCDriver driver= DriverBuilder.createDriver();
+ driver.execute(LeaseInfo.createTableSQL());
+ }
+
+ @Test
+ public void testLease() throws InterruptedException {
+ String leaseName="lease.test";
+ int leaseTime=5;
+ LeaseComponent.getInstance().getService().startLeaseTask(leaseName, leaseTime, new ILeaseGetCallback() {
+ @Override
+ public void callback(Date nextLeaseDate) {
+ System.out.println("I get lease");
+ }
+ });
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ }
+
+ @Test
+ public void testLock() throws InterruptedException {
+ String name="dipper";
+ String lockName="lease.test";
+ int leaseTime=5;
+ boolean success=LeaseComponent.getInstance().getService().lock(name,lockName,leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(6000);
+ assertTrue(!LeaseComponent.getInstance().getService().hasHoldLock(name,lockName));//超期释放
+ }
+
+ /**
+ * holdlock是一直持有锁，和租约的区别是，当释放锁后，无其他实例抢占
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testHoldLock() throws InterruptedException {
+ String name="dipper";
+ String lockName="lease.test";
+ int leaseTime=6;
+ boolean success=LeaseComponent.getInstance().getService().holdLock(name,lockName,leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(8000);
+ assertTrue(LeaseComponent.getInstance().getService().hasHoldLock(name,lockName));//会自动续约，不会释放，可以手动释放
+ LeaseComponent.getInstance().getService().unlock(name,lockName);
+ assertTrue(!LeaseComponent.getInstance().getService().hasHoldLock(name,lockName));
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
new file mode 100644
index 0000000..a8ba9ab
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.junit.Test;
+
+public class ORMUtilTest {
+ private String URL="";
+ protected String USER_NAME="";
+ protected String PASSWORD="";
+
+ public ORMUtilTest(){
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL,URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME,USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD,PASSWORD);//password
+ }
+ @Test
+ public void testInsert(){
+ String namespace="org.apache.configuable.test";
+ List<Person> personList=new ArrayList<>();
+ for(int i=0;i<10;i++){
+ personList.add(createPerson(namespace,"chris"+i));
+ }
+ /**
+ * 不带数据库连接信息（url,userName,Password），使用ConfiguableComponet的连接信息
+ */
+ ORMUtil.batchIgnoreInto(personList);//批量插入，如果有唯一键冲突，替换
+ ORMUtil.batchIgnoreInto(personList);//批量插入，如果有唯一键冲突，忽略
+ ORMUtil.batchInsertInto(personList);////批量插入，如果有唯一键冲突，跑错
+ }
+
+ @Test
+ public void testQueryList(){
+ Map<String,Integer> paras=new HashMap<>();
+ paras.put("age",18);
+ List<Person> personList=ORMUtil.queryForList("select * from person where age >${age} limit 100",paras,Person.class);
+ }
+
+ @Test
+ public void testQueryOneRow(){
+ Person personPara=new Person();
+ personPara.setAge(18);
+ personPara.setName("chris1");
+ Person person=ORMUtil.queryForObject("select * from person where age =${age} and name='${name}' ",personPara,Person.class,URL,USER_NAME,PASSWORD);
+ }
+
+ /**
+ * 创建configuable对象
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace,String name){
+ Person person=new Person();
+ person.setName(name);
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
+
+
+class Person extends BasedConfigurable {
+ @ENVDependence
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "org.apache.rocketmq.streams.Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+
+ @Override
+ public Object clone() {
+ Person person = null;
+ try {
+ person = (Person)super.clone();
+ } catch (CloneNotSupportedException e) {
+ System.out.println("clone error " + e);
+ }
+ return person;
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/SplitTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/SplitTest.java
new file mode 100644
index 0000000..56ab5ae
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/SplitTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.SplitStream;
+import org.apache.rocketmq.streams.common.functions.FilterFunction;
+import org.apache.rocketmq.streams.common.functions.SplitFunction;
+import org.junit.Test;
+
+public class SplitTest implements Serializable {
+
+ @Test
+ public void testUnion() {
+ DataStream stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ SplitStream splitStream = stream.split(new SplitFunction<JSONObject>() {
+ @Override
+ public String split(JSONObject o) {
+ if (o.getInteger("age") < 18) {
+ return "children";
+ } else if (o.getInteger("age") >= 18) {
+ return "adult";
+ }
+ return null;
+ }
+ });
+
+ DataStream children = splitStream.select("children");
+ DataStream adult = splitStream.select("adult");
+ children.union(adult)
+ .toPrint()
+ .start();
+
+ }
+
+ @Test
+ public void testDim() {
+ DataStreamAction stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }))
+ .join("dburl", "dbUserName", "dbPassowrd", "tableNameOrSQL", 5)
+ .setCondition("(name,==,name)")
+ .toDataSteam()
+ .selectFields("name", "age", "address")
+ .toPrint();
+
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/UnionTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/UnionTest.java
new file mode 100644
index 0000000..1b49185
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/UnionTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.functions.FilterFunction;
+import org.junit.Test;
+
+public class UnionTest implements Serializable {
+
+ @Test
+ public void testUnion() {
+ DataStream leftStream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ DataStream rightStream = (StreamBuilder.dataStream("namespace", "name2")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ leftStream.union(rightStream).toPrint().start();
+ }
+
+ @Test
+ public void testDim() {
+ DataStreamAction stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }))
+ .join("dburl", "dbUserName", "dbPassowrd", "tableNameOrSQL", 5)
+ .setCondition("(name,==,name)")
+ .toDataSteam()
+ .selectFields("name", "age", "address")
+ .toPrint();
+
+ stream.start();
+
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/WindowTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/WindowTest.java
new file mode 100644
index 0000000..69d98b2
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/WindowTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+
+import java.io.Serializable;
+
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.client.transform.window.TumblingWindow;
+import org.apache.rocketmq.streams.common.functions.ForEachFunction;
+import org.apache.rocketmq.streams.common.functions.MapFunction;
+import org.junit.Test;
+
+public class WindowTest implements Serializable {
+
+ @Test
+ public void testWindow() {
+ StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/duheng/project/opensource/sls_100.txt", false)
+ .map((MapFunction<JSONObject, String>)message -> JSONObject.parseObject(message))
+ .window(TumblingWindow.of(Time.seconds(5)))
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(true)
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "InFlow")
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+ protected int sum = 0;
+
+ @Override
+ public void foreach(JSONObject o) {
+ int total = o.getInteger("total");
+ sum = sum + total;
+ o.put("sum(total)", sum);
+ }
+ }).toPrint().start();
+
+ }
+
+ // @Test
+ // public void testWindowFromMetaq() throws InterruptedException {
+ // String topic = "TOPIC_DIPPER_SYSTEM_MSG_4";
+ // StreamBuilder.dataStream("namespace", "name")
+ // .fromFile("/Users/yuanxiaodong/chris/sls_100.txt", true)
+ // .toRocketmq(topic)
+ // .asyncStart();
+ //
+ // StreamBuilder.dataStream("namespace", "name1")
+ // .fromRocketmq(topic, "chris", true)
+ // .window(TumblingWindow.of(Time.seconds(5)))
+ // .groupby("ProjectName", "LogStore")
+ // .setLocalStorageOnly(true)
+ // .count("total")
+ // .sum("OutFlow", "OutFlow")
+ // .sum("InFlow", "inflow")
+ // .toDataSteam()
+ // .forEach(new ForEachFunction<JSONObject>() {
+ // protected int sum = 0;
+ //
+ // @Override
+ // public void foreach(JSONObject o) {
+ // int total = o.getInteger("total");
+ // sum = sum + total;
+ // o.put("sum(total)", sum);
+ // }
+ // }).toPrint().start();
+ // }
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/AbstractWindowFireModeTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/AbstractWindowFireModeTest.java
new file mode 100644
index 0000000..c102f1f
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/AbstractWindowFireModeTest.java
@@ -0,0 +1,189 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.client.transform.window.TumblingWindow;
+import org.apache.rocketmq.streams.common.functions.ForEachFunction;
+import org.apache.rocketmq.streams.common.functions.MapFunction;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+
+
+public abstract class AbstractWindowFireModeTest implements Serializable {
+ protected Date date=new Date();
+ public AbstractWindowFireModeTest(){
+
+ date.setYear(2021-1900);
+ date.setMonth(6);
+ date.setDate(14);
+ date.setHours(12);
+ date.setMinutes(1);
+ date.setSeconds(0);
+
+ }
+
+ public void testWindowFireMode0(boolean isLocalOnly) throws InterruptedException {
+ testWindowFireMode1(isLocalOnly,5);
+ }
+
+ public void testWindowFireMode0(boolean isLocalOnly,int windowSize) throws InterruptedException {
+
+ createSourceDataStream().map(new MapFunction<JSONObject, String>() {
+ int count=0;
+ Long time=null;
+ @Override
+ public JSONObject map(String message) throws Exception {
+
+ if(time==null){
+ time=date.getTime();
+ }else {
+ time+=count;
+ }
+ count++;
+ JSONObject msg=JSONObject.parseObject(message);
+
+ msg.put("logTime",time);
+
+ return msg;
+ }
+ })
+ .window(TumblingWindow.of(Time.seconds(windowSize)))
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(isLocalOnly)
+ .setMaxMsgGap(isLocalOnly?10L:20L)
+ .setTimeField("logTime")
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "inflow")
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+ AtomicInteger sum = new AtomicInteger(0) ;
+ @Override
+ public synchronized void foreach(JSONObject o) {
+ int total = o.getInteger("total");
+ o.put("sum(total)", sum.addAndGet(total));
+ }
+ }).toPrint().start();
+ }
+ public void testWindowFireMode1(boolean isLocalOnly) throws InterruptedException {
+ testWindowFireMode1(isLocalOnly,5);
+ }
+ public void testWindowFireMode1(boolean isLocalOnly,int windowSize) throws InterruptedException {
+ AtomicInteger sum = new AtomicInteger(0) ;
+ createSourceDataStream()
+ //.map(new MapFunction<JSONObject, String>() {
+ // AtomicInteger COUNT=new AtomicInteger(0);
+ // Long time;
+ // @Override
+ // public JSONObject map(String message) throws Exception {
+ //
+ // if(time==null){
+ // time=date.getTime();
+ // }else {
+ // int count=COUNT.incrementAndGet();
+ // time+=count;
+ // }
+ // JSONObject msg=JSONObject.parseObject(message);
+ //
+ // msg.put("logTime",time);
+ // return msg;
+ // }
+ //})
+ .window(TumblingWindow.of(Time.seconds(windowSize)))
+ .setTimeField("logTime")
+ .fireMode(1)
+ .setMaxMsgGap(isLocalOnly?20L:20L)
+ .waterMark(100000000)
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(isLocalOnly)
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "InFlow")
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+
+
+ @Override
+ public synchronized void foreach(JSONObject o) {
+ int total = o.getInteger("total");
+ o.put("sum(total)", sum.addAndGet(total));
+ }
+ }).toPrint().start();
+ }
+
+ public void testWindowFireMode2(boolean isLocalOnly){
+ long time=new Date().getTime();
+ System.out.println(DateUtil.getCurrentTimeString());
+ createSourceDataStream()
+ .map(new MapFunction<JSONObject, String>() {
+ int count=0;
+ @Override
+ public JSONObject map(String message) throws Exception {
+
+ JSONObject msg=JSONObject.parseObject(message);
+ long time= msg.getLong("logTime");
+ Date date=new Date(time);
+ date.setYear(2021-1900);
+ date.setMonth(6);
+ date.setDate(14);
+ msg.put("logTime",date.getTime()+count++);
+ return msg;
+ }
+ })
+ .window(TumblingWindow.of(Time.seconds(5)))
+ .setTimeField("logTime")
+ .setMaxMsgGap(isLocalOnly?5L:20L)
+ .fireMode(1)
+ .waterMark(100000000)
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(isLocalOnly)
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "InFlow")
+ .toDataSteam()
+ .map(new MapFunction<JSONObject, JSONObject>() {
+ long time=new Date().getTime();
+ @Override
+ public JSONObject map(JSONObject message) throws Exception {
+ message.put("name","chris");
+ message.put("time",time++);
+ return message;
+ }
+ })
+ .window(TumblingWindow.of(Time.seconds(5)))
+ .fireMode(2).waterMark(100000000)
+ .setMaxMsgGap(80L)
+ .groupBy("name")
+ .setTimeField("time")
+ .sum("total","sum_total")
+ .setLocalStorageOnly(true)
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+ AtomicInteger sum = new AtomicInteger(0) ;
+ Map<String,Integer> map=new HashMap<>();
+ @Override
+ public synchronized void foreach(JSONObject o) {
+ String windowInstanceId=o.getString("windowInstanceId");
+ Integer oldValue=map.get(windowInstanceId);
+ int total = o.getInteger("sum_total");
+ if(oldValue!=null){
+ total=total-oldValue;
+ }
+ int nowValue=sum.addAndGet(total);
+ map.put(windowInstanceId,total);
+ o.put("sum(total)", nowValue);
+ }
+ }).toPrint().start();
+
+ }
+
+
+ protected abstract DataStream createSourceDataStream();
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromFileTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromFileTest.java
new file mode 100644
index 0000000..71a2593
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromFileTest.java
@@ -0,0 +1,158 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.PrintUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.batchloader.IRowOperator;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.apache.rocketmq.streams.window.model.WindowInstance;
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+import org.apache.rocketmq.streams.window.operator.impl.WindowOperator;
+import org.apache.rocketmq.streams.window.state.impl.WindowValue;
+import org.apache.rocketmq.streams.window.storage.WindowStorage;
+import org.junit.Test;
+
+public class WindowFromFileTest extends AbstractWindowFireModeTest {
+ protected DataStream createSourceDataStream(){
+ return StreamBuilder.dataStream("namespace", "name1")
+ .fromFile("/Users/yuanxiaodong/chris/sls_100000.txt",false);
+ }
+
+ /**
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWindowFireMode0AndNoTimeField() throws InterruptedException {
+ super.testWindowFireMode0(false);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode1() throws InterruptedException {
+ super.testWindowFireMode1(false);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode2() {
+ super.testWindowFireMode2(false);
+ }
+
+
+
+ @Test
+ public void testSLS10(){
+ Map<String,Integer> map=create();
+
+ System.out.println(map.size());
+ PrintUtil.print(map);
+ }
+
+
+ @Test
+ public void testLoadData() throws InterruptedException {
+ Map<String,Integer> map=create();
+ WindowOperator windowOperator=new WindowOperator();
+ windowOperator.setNameSpace("namespace");
+ windowOperator.setConfigureName("name1");
+ WindowInstance windowInstance=windowOperator.createWindowInstance("2021-07-14 02:00:00","2021-07-14 03:00:00","2021-07-14 03:00:00","1");
+ WindowStorage storage= new WindowStorage();
+ storage.loadSplitData2Local("1", windowInstance.createWindowInstanceId(), WindowValue.class, new IRowOperator() {
+ @Override
+ public void doProcess(Map<String, Object> row) {
+ WindowValue windowValue = ORMUtil.convert(row, WindowValue.class);
+ String groupBy=windowValue.getGroupBy();
+ if(!map.containsKey(groupBy)){
+
+ System.out.println(groupBy+" ======");
+ }else {
+ Integer count=(Integer)windowValue.getComputedColumnResultByKey("total");
+ int mapCount=map.get(groupBy);
+ if(count!=mapCount){
+ System.out.println(groupBy+" ======");
+ }
+ }
+ }
+ });
+
+
+ Thread.sleep(10000000l);
+ }
+
+
+ protected Map<String,Integer> create(){
+ List<String> lines= FileUtil.loadFileLine("/Users/yuanxiaodong/chris/sls_10000.txt");
+ Map<String,Integer> map=new HashMap<>();
+ for(String line:lines){
+ JSONObject msg=JSONObject.parseObject(line);
+ String projectName=msg.getString("ProjectName");
+ String logstore=msg.getString("LogStore");
+
+ String key= MapKeyUtil.createKey(projectName,logstore);
+
+ if (StringUtil.isEmpty(key)) {
+ key="<null>";
+ }
+ Integer count=map.get(key);
+ if(count==null){
+ map.put(key,1);
+ }else {
+ count++;
+ map.put(key,count);
+ }
+ }
+ return map;
+ }
+
+
+ @Test
+ public void testFileResult(){
+ createFile(3);
+ }
+
+
+ protected void createFile(int count){
+
+ date.setYear(2021-1900);
+ date.setMonth(6);
+ date.setDate(27);
+ date.setHours(12);
+ date.setMinutes(1);
+ date.setSeconds(0);
+ Long time=null;
+ for(int i=0;i<count;i++){
+ List<String> lines= FileUtil.loadFileLine("/Users/yuanxiaodong/chris/sls_100000.txt");
+ List<String> msgs=new ArrayList<>();
+ for(String line:lines){
+ JSONObject msg=JSONObject.parseObject(line);
+ if(time==null){
+ time=date.getTime();
+ }else {
+ time=time+1;
+ }
+ msg.put("logTime",time);
+ AbstractWindow window=new WindowOperator();
+ window.setSizeInterval(5);
+ window.setTimeUnitAdjust(1);
+ window.setTimeFieldName("logTime");
+ //WindowInstance windowInstance= window.queryOrCreateWindowInstance(new Message(msg),"1").get(0);
+ msgs.add(msg.toJSONString());
+ }
+ FileUtil.write("/Users/yuanxiaodong/chris/sls_100000_time_"+i+".txt",msgs);
+ }
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromMetaq.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromMetaq.java
new file mode 100644
index 0000000..235ce3d
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromMetaq.java
@@ -0,0 +1,47 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.junit.Test;
+
+public class WindowFromMetaq extends AbstractWindowFireModeTest {
+
+ String topic = "TOPIC_DIPPER_SYSTEM_MSG_5";
+ @Test
+ public void testWindowFireMode0() throws InterruptedException {
+ super.testWindowFireMode0(true);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode1() throws InterruptedException {
+ super.testWindowFireMode1(true);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode2() {
+ super.testWindowFireMode2(true);
+ }
+
+
+ @Test
+ public void testWindowToMetaq() throws InterruptedException {
+
+ long start=System.currentTimeMillis();
+ StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_10.txt", true)
+ .toMetaq(topic)
+ .start();
+ }
+
+
+ protected DataStream createSourceDataStream(){
+ return StreamBuilder.dataStream("namespace", "name1")
+ .fromRocketmq(topic,"chris1",true);
+ }
+
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowHighAvailabilityTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowHighAvailabilityTest.java
new file mode 100644
index 0000000..cbca7bb
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowHighAvailabilityTest.java
@@ -0,0 +1,131 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.batchloader.BatchRowLoader;
+import org.apache.rocketmq.streams.db.driver.batchloader.IRowOperator;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.apache.rocketmq.streams.window.model.WindowInstance;
+import org.apache.rocketmq.streams.window.operator.impl.WindowOperator;
+import org.apache.rocketmq.streams.window.state.impl.WindowValue;
+import org.apache.rocketmq.streams.window.storage.WindowStorage;
+import org.junit.Test;
+
+public class WindowHighAvailabilityTest extends AbstractWindowFireModeTest {
+ protected DataStream createSourceDataStream(){
+ return StreamBuilder.dataStream("namespace", "name1")
+ .fromFile("/Users/yuanxiaodong/chris/sls_100000.txt",false);
+ }
+
+ /**
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWindowFireMode0AndNoTimeField() throws InterruptedException {
+ super.testWindowFireMode0(false,60*60);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode1() throws InterruptedException {
+ super.testWindowFireMode1(false,60*5);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode2() {
+ super.testWindowFireMode2(false);
+ }
+
+ @Test
+ public void testLoadData() throws InterruptedException {
+ Map<String,Integer> map=create();
+ WindowOperator windowOperator=new WindowOperator();
+ windowOperator.setNameSpace("namespace");
+ windowOperator.setConfigureName("name1_window_10001");
+ WindowInstance windowInstance=windowOperator.createWindowInstance("2021-07-14 02:00:00","2021-07-14 03:00:00","2021-07-14 03:00:00","1");
+
+ WindowStorage storage= new WindowStorage();
+ System.out.println("1;namespace;name1_window_10001;name1_window_10001;2021-07-14 02:00:00;2021-07-14 03:00:00");
+ System.out.println(windowInstance.createWindowInstanceId());
+ System.out.println(StringUtil.createMD5Str(windowInstance.createWindowInstanceId()));
+ Map<String, WindowValue> windowValueMap=new HashMap<>();
+ AtomicInteger sum=new AtomicInteger(0);
+ BatchRowLoader batchRowLoader = new BatchRowLoader("partition_num",
+ "select * from " + ORMUtil.getTableName(WindowValue.class) + " where window_instance_partition_id ='"
+ + StringUtil.createMD5Str(windowInstance.createWindowInstanceId() )+ "'", new IRowOperator(){
+
+ @Override
+ public void doProcess(Map<String, Object> row) {
+ WindowValue windowValue = ORMUtil.convert(row, WindowValue.class);
+
+ String groupBy=windowValue.getGroupBy();
+ windowValueMap.put(groupBy,windowValue);
+ if(!map.containsKey(groupBy)){
+
+ System.out.println(groupBy+" ======");
+ }else {
+ Integer count=(Integer)windowValue.getComputedColumnResultByKey("total");
+ int mapCount=map.get(groupBy);
+ if(count!=mapCount){
+ System.out.println(groupBy+" ======");
+ }
+ //System.out.println(groupBy+" "+count);
+ sum.addAndGet(count);
+
+ }
+ }
+ });
+ batchRowLoader.startLoadData();
+
+ System.out.println("sum "+sum.get());
+ for(String key:map.keySet()){
+ if(windowValueMap.containsKey(key)==false){
+ System.out.println(key+" "+map.get(key));
+ }
+ }
+
+ Thread.sleep(10000000l);
+ }
+
+
+ protected Map<String,Integer> create(){
+ List<String> lines= FileUtil.loadFileLine("/Users/yuanxiaodong/chris/sls_10000.txt");
+ Map<String,Integer> map=new HashMap<>();
+ for(String line:lines){
+ JSONObject msg=JSONObject.parseObject(line);
+ String projectName=msg.getString("ProjectName");
+ String logstore=msg.getString("LogStore");
+
+ String key= MapKeyUtil.createKey(projectName,logstore);
+
+ if (StringUtil.isEmpty(key)) {
+ key="<null>";
+ }
+ Integer count=map.get(key);
+ if(count==null){
+ map.put(key,1);
+ }else {
+ count++;
+ map.put(key,count);
+ }
+ }
+ return map;
+ }
+
+
+
+}
diff --git a/rocketmq-streams-clients/src/test/resources/log4j.xml b/rocketmq-streams-clients/src/test/resources/log4j.xml
new file mode 100755
index 0000000..a1afd37
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/resources/log4j.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
diff --git a/rocketmq-streams-dim/pom.xml b/rocketmq-streams-dim/pom.xml
new file mode 100644
index 0000000..1175370
--- /dev/null
+++ b/rocketmq-streams-dim/pom.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-dim</artifactId>
+ <name>ROCKETMQ STREAMS :: dim</name>
+ <packaging>jar</packaging>
+ <properties>
+ <file_encoding>UTF-8</file_encoding>
+ <project.build.sourceEncoding>${file_encoding}</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-filter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-http</artifactId>
+ </dependency>
+ </dependencies>
+
+
+</project>
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/DimComponent.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/DimComponent.java
new file mode 100644
index 0000000..8602b5d
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/DimComponent.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim;
+
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.dim.service.IDimService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.dim.service.impl.DimServiceImpl;
+
+public class DimComponent extends AbstractComponent<IDimService> {
+
+ private static final Log LOG = LogFactory.getLog(DimComponent.class);
+
+ // private transient Map<String, DBDim> nameListMap = new HashMap<>();
+
+ protected transient ConfigurableComponent configurableComponent;
+ private transient IDimService dimService;
+
+ public static DimComponent getInstance(String namespace) {
+ return ComponentCreator.getComponent(namespace, ComponentCreator.class);
+ }
+
+ @Override
+ protected boolean startComponent(String namespace) {
+ configurableComponent = ComponentCreator.getComponent(namespace, ConfigurableComponent.class);
+ dimService = new DimServiceImpl(configurableComponent);
+ return true;
+ }
+
+ @Override
+ protected boolean initProperties(Properties properties) {
+ return true;
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ @Override
+ public IDimService getService() {
+ return dimService;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/builder/DimBuilder.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/builder/DimBuilder.java
new file mode 100644
index 0000000..508bb19
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/builder/DimBuilder.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.builder;
+
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+
+public class DimBuilder {
+
+ private String url;
+ private String password;
+ private String userName;
+ protected Long pollingTime = 60L; // 同步数据的事件间隔
+ private String jdbcDriver = "com.mysql.jdbc.Driver";
+
+ public DimBuilder(String url, String userName, String password) {
+ this.url = url;
+ this.password = password;
+ this.userName = userName;
+ }
+
+ public DBDim createDim(String namespace, String name, String sqlOrTableName) {
+ DBDim nameList = new DBDim();
+ nameList.setNameSpace(namespace);
+ if (StringUtil.isNotEmpty(name)) {
+ nameList.setConfigureName(name);
+ }
+ String sql = sqlOrTableName;
+ if (sqlOrTableName.split(" ").length == 1) {
+ sql = "select * from " + sqlOrTableName + " limit 500000";
+ }
+ nameList.setSql(sql);
+ nameList.setJdbcdriver(jdbcDriver);
+ nameList.setPollingTimeMintue(pollingTime);
+ nameList.setUrl(url);
+ nameList.setUserName(userName);
+ nameList.setPassword(password);
+ return nameList;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public Long getPollingTime() {
+ return pollingTime;
+ }
+
+ public void setPollingTime(Long pollingTime) {
+ this.pollingTime = pollingTime;
+ }
+
+ public String getJdbcDriver() {
+ return jdbcDriver;
+ }
+
+ public void setJdbcDriver(String jdbcDriver) {
+ this.jdbcDriver = jdbcDriver;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/InExpressionResource.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/InExpressionResource.java
new file mode 100644
index 0000000..c8a13a7
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/InExpressionResource.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.expression;
+
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.apache.rocketmq.streams.filter.context.RuleContext;
+import org.apache.rocketmq.streams.filter.function.expression.AbstractExpressionFunction;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethodAilas;
+
+import java.util.Map;
+
+@Function
+public class InExpressionResource extends AbstractExpressionFunction {
+
+ /**
+ * value格式 :resourceName.fieldName。如果只有单列，可以省略.fieldname
+ *
+ * @param expression
+ * @param context
+ * @param rule
+ * @return
+ */
+ @FunctionMethod(value = "in_expression_resouce", alias = "in_resouce")
+ @FunctionMethodAilas("in_expression_resouce(resourceName->(varName,functionName,value)&((varName,functionName,"
+ + "value)|(varName,functionName,value)))")
+ @Override
+ public Boolean doExpressionFunction(Expression expression, RuleContext context, Rule rule) {
+ return match(expression, context, rule, false);
+ }
+
+ protected Boolean match(Expression expression, RuleContext context, Rule rule, boolean supportRegex) {
+ Object value = expression.getValue();
+ if (value == null) {
+ return null;
+ }
+ String valueStr = String.valueOf(value);
+ String[] valueArray = valueStr.split("->");
+ String dataResourceNamespace = rule.getNameSpace();
+ String dataResourceName = null;
+ String expressionStr = null;
+ if (valueArray.length == 2) {
+ dataResourceName = valueArray[0];
+ expressionStr = valueArray[1];
+ }
+ if (valueArray.length > 2) {
+ dataResourceNamespace = valueArray[0];
+ dataResourceName = valueArray[1];
+ expressionStr = valueArray[2];
+ }
+
+ DBDim dataResource =
+ (DBDim)context.getConfigurableService().queryConfigurableByIdent(DBDim.TYPE, dataResourceName);
+ if (dataResource == null) {
+ return null;
+ }
+ Map<String, Object> row = dataResource.matchExpression(expressionStr, context.getMessage().getMessageBody());
+ if (row != null && row.size() > 0) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/NotInExpressionResource.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/NotInExpressionResource.java
new file mode 100644
index 0000000..752276c
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/NotInExpressionResource.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.expression;
+
+import org.apache.rocketmq.streams.filter.context.RuleContext;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethodAilas;
+
+@Function
+public class NotInExpressionResource extends InExpressionResource {
+
+ /**
+ * value格式 :resourceName.fieldName。如果只有单列，可以省略.fieldname
+ *
+ * @param expression
+ * @param context
+ * @param rule
+ * @return
+ */
+ @FunctionMethod("not_in_expression_resouce")
+ @FunctionMethodAilas("not_in_expression_resouce(resourceName->(varName,functionName,value)&((varName,"
+ + "functionName,value)|(varName,functionName,value)))")
+ @Override
+ public Boolean doExpressionFunction(Expression expression, RuleContext context, Rule rule) {
+ return !match(expression, context, rule, false);
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceFunction.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceFunction.java
new file mode 100644
index 0000000..c2ffb88
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceFunction.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.script;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.dim.DimComponent;
+import org.apache.rocketmq.streams.dim.intelligence.AbstractIntelligenceCache;
+import org.apache.rocketmq.streams.script.context.FunctionContext;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.script.utils.FunctionUtils;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+@Function
+public class IntelligenceFunction {
+
+ private static final Log LOG = LogFactory.getLog(IntelligenceFunction.class);
+
+ private DimComponent nameListComponent;
+
+ @FunctionMethod(value = "intelligence", alias = "qingbao")
+ public void intelligence(IMessage message, FunctionContext context, String namespace, String nameListName, String intelligenceFieldName, String asName) {
+ intelligenceInner(message, context, namespace, nameListName, intelligenceFieldName, asName, true);
+ }
+
+ @FunctionMethod(value = "left_join_intelligence", alias = "left_join_qingbao")
+ public void intelligenceLeftJoin(IMessage message, FunctionContext context, String namespace, String nameListName, String intelligenceFieldName, String asName) {
+ intelligenceInner(message, context, namespace, nameListName, intelligenceFieldName, asName, false);
+ }
+
+ public void intelligenceInner(IMessage message, FunctionContext context, String namespace, String nameListName, String intelligenceFieldName, String asName, boolean isInner) {
+ String key = FunctionUtils.getValueString(message, context, intelligenceFieldName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ ConfigurableComponent configurableComponent = ComponentCreator.getComponent(namespace, ConfigurableComponent.class);
+ AbstractIntelligenceCache intelligenceCache = configurableComponent.queryConfigurable(AbstractIntelligenceCache.TYPE, nameListName);
+ if (intelligenceCache == null) {
+ throw new RuntimeException("can not query intelligence. the namespace is " + namespace + ", the name is " + nameListName);
+ }
+ Map<String, Object> row = intelligenceCache.getRow(key);
+ if (row != null) {
+ asName = FunctionUtils.getValueString(message, context, asName);
+ if (StringUtil.isNotEmpty(asName)) {
+ asName = asName + ".";
+ } else {
+ asName = "";
+ }
+ for (Entry<String, Object> entry : row.entrySet()) {
+ String elementKey = asName + entry.getKey();
+ message.getMessageBody().put(elementKey, entry.getValue());
+ }
+ } else {
+ if (isInner) {
+ context.breakExecute();
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceNameListFunction.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceNameListFunction.java
new file mode 100644
index 0000000..02d67fb
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceNameListFunction.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.script;
+
+import org.apache.rocketmq.streams.script.annotation.Function;
+
+@Function
+public class IntelligenceNameListFunction {
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/NameListFunction.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/NameListFunction.java
new file mode 100644
index 0000000..3b03a4e
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/NameListFunction.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.script;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.dim.DimComponent;
+import org.apache.rocketmq.streams.script.context.FunctionContext;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.script.utils.FunctionUtils;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+@Function
+public class NameListFunction {
+
+ private static final Log LOG = LogFactory.getLog(NameListFunction.class);
+
+ private DimComponent nameListComponent;
+
+ @FunctionMethod(value = "innerJoin", alias = "inner_join")
+ public JSONArray innerJoin(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String alias, String script, String... fieldNames) {
+ JSONArray rows = getRows(message, context, namespace, nameListName, expressionStr, alias, script, fieldNames);
+ if (rows == null || rows.size() == 0) {
+ context.breakExecute();
+ return null;
+ }
+ return rows;
+ }
+
+ @FunctionMethod(value = "leftJoin", alias = "left_join")
+ public JSONArray leftJoin(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String alias, String script, String... fieldNames) {
+ JSONArray rows = getRows(message, context, namespace, nameListName, expressionStr, alias, script, fieldNames);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ return rows;
+ }
+
+ @FunctionMethod(value = "mark_rows", alias = "namelist_rows")
+ public String namelist(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String... fieldNames) {
+ JSONArray rows = getRows(message, context, namespace, nameListName, expressionStr, null, null, fieldNames);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ return rows.toJSONString();
+ }
+
+ @FunctionMethod(value = "mark", alias = "namelist")
+ public String namelist(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String fieldName) {
+ String tmp = fieldName;
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ expressionStr = FunctionUtils.getValueString(message, context, expressionStr);
+ fieldName = FunctionUtils.getValueString(message, context, fieldName);
+ if (fieldName == null) {
+ fieldName = tmp;
+ }
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ Map<String, Object> row =
+ nameListComponent.getService().match(nameListName, expressionStr, message.getMessageBody());
+ if (row != null) {
+ Object value = row.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ return value.toString();
+ }
+ return null;
+ }
+
+ @FunctionMethod(value = "mark", alias = "namelist")
+ public String namelist(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String fieldNames, String joinMark) {
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ expressionStr = FunctionUtils.getValueString(message, context, expressionStr);
+ fieldNames = FunctionUtils.getValueString(message, context, fieldNames);
+ joinMark = FunctionUtils.getValueString(message, context, joinMark);
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ Map<String, Object> row =
+ nameListComponent.getService().match(nameListName, expressionStr, message.getMessageBody());
+ if (row != null) {
+ String[] fieldNameTem = fieldNames.split(",");
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < fieldNameTem.length; i++) {
+ Object tem = row.get(fieldNameTem[i]);
+ if (tem != null) {
+ if (result.length() == 0) {
+ if (StringUtil.isNotEmpty(tem.toString()) && !("null".equalsIgnoreCase(tem.toString()))) {
+ result.append(tem);
+ }
+ } else {
+ if (StringUtil.isNotEmpty(tem.toString()) && !("null".equalsIgnoreCase(tem.toString()))) {
+ result.append(joinMark + tem);
+ }
+ }
+ }
+
+ }
+ return result.toString();
+ }
+ return null;
+ }
+
+ @FunctionMethod(value = "in_namelist", alias = "in_namelist")
+ public Boolean inNameList(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr) {
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ Map<String, Object> row =
+ nameListComponent.getService().match(nameListName, expressionStr, message.getMessageBody());
+ if (row != null && row.size() > 0) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * 根据表达式，从namelist中获取符合条件的数据
+ *
+ * @param message
+ * @param context
+ * @param namespace
+ * @param nameListName
+ * @param expressionStr （varname,functionName,value)&(varname,functionName,value)
+ * @param fieldNames 需要返回的字段名
+ * @return
+ */
+ protected JSONArray getRows(IMessage message, FunctionContext context, String namespace, String nameListName, String expressionStr, String alias, String script, String... fieldNames) {
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ expressionStr = FunctionUtils.getValueString(message, context, expressionStr);
+ script = FunctionUtils.getValueString(message, context, script);
+ if (StringUtil.isEmpty(script)) {
+ script = null;
+ }
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ List<Map<String, Object>> rows =
+ nameListComponent.getService().matchSupportMultiRow(nameListName, expressionStr, message.getMessageBody(), script);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ JSONArray jsonArray = new JSONArray();
+ for (Map<String, Object> row : rows) {
+ JSONObject jsonObject = new JSONObject();
+ if (fieldNames == null || fieldNames.length == 0) {
+ if (alias == null) {
+ jsonObject.putAll(row);
+ } else {
+ Iterator<Entry<String, Object>> it = row.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<String, Object> entry = it.next();
+ String fieldName = entry.getKey();
+ if (alias != null) {
+ fieldName = alias + "." + fieldName;
+ }
+ jsonObject.put(fieldName, entry.getValue());
+ }
+ }
+
+ } else {
+ for (String fieldName : fieldNames) {
+ String tmp = fieldName;
+ if (alias != null) {
+ fieldName = alias + "." + fieldName;
+ }
+ jsonObject.put(fieldName, row.get(tmp));
+ }
+ }
+ jsonArray.add(jsonObject);
+ }
+ return jsonArray;
+ }
+
+}
\ No newline at end of file
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/DimIndex.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/DimIndex.java
new file mode 100644
index 0000000..6c6aeef
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/DimIndex.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.index;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.datatype.IntDataType;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.rocketmq.streams.common.cache.CompressTable;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DimIndex {
+
+ private static final Log LOG = LogFactory.getLog(DimIndex.class);
+
+ /**
+ * 索引字段名，支持多个索引，每个索引一行，支持组合索引，多个字段用；拼接 name 单索引 name;age 组合索引
+ */
+ protected List<String> indexs = new ArrayList<>();
+
+ /**
+ * 如果是唯一索引，用压缩值存储 每个索引一行
+ */
+ protected Map<String, IntValueKV> uniqueIndex;
+
+ /**
+ * 如果是非唯一索引，用这个结构存储 每个索引一行，后面的map：key：索引值；value：row id 列表，rowid用字节表示
+ */
+ protected Map<String, Map<String, List<Integer>>> mutilIndex = new HashMap<>();
+
+ protected boolean isUnique = false;//如果是唯一索引，值为true
+
+ public DimIndex(List<String> indexs) {
+ this.indexs = formatIndexs(indexs);
+ }
+
+ public DimIndex(String index, String... indexs) {
+ if (indexs == null) {
+ return;
+ }
+ List<String> indexList = new ArrayList<>();
+ for (String idx : indexs) {
+ indexList.add(idx);
+ }
+ this.indexs = formatIndexs(indexList);
+ }
+
+ /**
+ * 组合索引，多个字段要以名称顺序排列，完成索引名称的标注化处理
+ *
+ * @param indexs
+ * @return
+ */
+ protected List<String> formatIndexs(List<String> indexs) {
+ List<String> allIndex = new ArrayList<>();
+ for (String indexName : indexs) {
+ String[] values = indexName.split(";");
+ List<String> indexList = new ArrayList<>();
+ for (String value : values) {
+ indexList.add(value);
+ }
+ Collections.sort(indexList);
+ String indexKey = MapKeyUtil.createKey(indexList);
+ allIndex.add(indexKey);
+ }
+ return allIndex;
+ }
+
+ /**
+ * 加载一行数据，如果是唯一索引，在uniqueIndex中查找，否则在mutilIndex查找
+ *
+ * @param indexName 索引名，如name
+ * @param indexValue 索引值，如chris
+ * @return
+ */
+ public List<Integer> getRowIds(String indexName, String indexValue) {
+ if (isUnique) {
+ IntValueKV index = this.uniqueIndex.get(indexName);
+ if (index == null) {
+ return null;
+ }
+ Integer rowId = index.get(indexValue);
+ if (rowId == null) {
+ return null;
+ }
+ List<Integer> rowIds = new ArrayList<>();
+ rowIds.add(rowId);
+ return rowIds;
+ } else {
+ Map<String, List<Integer>> indexs = this.mutilIndex.get(indexName);
+ if (indexs == null) {
+ return null;
+ }
+ return indexs.get(indexValue);
+ }
+ }
+
+ /**
+ * 构建索引，如果是唯一索引，构建在uniqueIndex数据结构中，否则构建在mutilIndex这个数据结构中
+ *
+ * @param tableCompress 表数据
+ */
+ public void buildIndex(CompressTable tableCompress) {
+
+ if (isUnique) {
+ Map<String, IntValueKV> fieldIndex2RowIndex = new HashMap<>();
+ buildUniqueIndex(tableCompress, fieldIndex2RowIndex);
+ this.uniqueIndex = fieldIndex2RowIndex;
+ } else {
+ Map<String, Map<String, List<Integer>>> fieldIndex2RowIndex = new HashMap<>();
+ buildIndex(tableCompress, fieldIndex2RowIndex);
+ this.mutilIndex = fieldIndex2RowIndex;
+ }
+ }
+
+ /**
+ * 构建唯一索引
+ *
+ * @param tableCompress
+ * @param fieldIndex2RowIndex
+ */
+ protected void buildUniqueIndex(CompressTable tableCompress, Map<String, IntValueKV> fieldIndex2RowIndex) {
+ if (indexs == null || indexs.size() == 0) {
+ return;
+ }
+
+ int i = 0;
+ Iterator<Map<String, Object>> it = tableCompress.newIterator();
+ while (it.hasNext()) {
+ /**
+ * 为每个索引做构建
+ */
+ for (String indexName : indexs) {
+ IntValueKV index = fieldIndex2RowIndex.get(indexName);
+ if (index == null) {
+ synchronized (this) {
+ index = fieldIndex2RowIndex.get(indexName);
+ if (index == null) {
+ index = new IntValueKV(tableCompress.getRowCount());
+ fieldIndex2RowIndex.put(indexName, index);
+ }
+ }
+
+ }
+ String[] nameIndexs = indexName.split(";");
+ Arrays.sort(nameIndexs);
+ Map<String, String> cacheValues = createRow(it.next());
+ String indexValue = createIndexValue(cacheValues, nameIndexs);
+ index.put(indexValue, i);
+ }
+ i++;
+ }
+ LOG.info(" finish poll data , the row count is " + i + ". byte is " + tableCompress
+ .getByteCount());
+ }
+
+ /**
+ * 从table compress 中取出所有的行，构建索引。把
+ *
+ * @param dataCacheVar
+ * @param fieldIndex2RowIndex
+ */
+ protected void buildIndex(CompressTable dataCacheVar, Map<String, Map<String, List<Integer>>> fieldIndex2RowIndex) {
+ Iterator<Map<String, Object>> it = dataCacheVar.newIterator();
+ int i = 0;
+ while (it.hasNext()) {
+ Map<String, Object> row = it.next();
+ Map<String, String> cacheValues = createRow(row);
+ putValue2Index(cacheValues, i, fieldIndex2RowIndex);
+ i++;
+ }
+
+ LOG.info(" finish poll data , the row count is " + i + ". byte is " + dataCacheVar
+ .getByteCount());
+
+ }
+
+ /**
+ * 创建索引结构
+ *
+ * @param value
+ * @param rowIndex
+ * @param fieldIndex2RowIndex
+ */
+ protected void putValue2Index(Map<String, String> value, Integer rowIndex,
+ Map<String, Map<String, List<Integer>>> fieldIndex2RowIndex) {
+ if (indexs == null || indexs.size() == 0) {
+ return;
+ }
+ for (String indexName : indexs) {
+ Map<String, List<Integer>> name2RowIndexs = fieldIndex2RowIndex.get(indexName);
+ if (name2RowIndexs == null) {
+ synchronized (this) {
+ name2RowIndexs = fieldIndex2RowIndex.get(indexName);
+ if (name2RowIndexs == null) {
+ name2RowIndexs = new HashMap<>();
+ fieldIndex2RowIndex.put(indexName, name2RowIndexs);
+ }
+ }
+
+ }
+ String[] nameIndexs = indexName.split(";");
+ Arrays.sort(nameIndexs);
+ addValue2Index(indexName, name2RowIndexs, rowIndex, value, nameIndexs);
+ }
+ }
+
+ /**
+ * 根据索引名称，把不同的索引值创建key，value放入索引缓存
+ *
+ * @param indexName 索引名称，多字段索引用；分割
+ * @param name2RowIndexs 索引值，list<id>
+ * @param rowIndex 行号
+ * @param row
+ */
+ protected void addValue2Index(String indexName, Map<String, List<Integer>> name2RowIndexs, Integer rowIndex,
+ Map<String, String> row, String[] nameIndexs) {
+ String indexValue = createIndexValue(row, nameIndexs);
+ addValue2Index(name2RowIndexs, rowIndex, indexValue);
+ }
+
+ /**
+ * 对于组合索引，把各个字段的值取出来
+ *
+ * @param row
+ * @param nameIndexs
+ * @return
+ */
+ protected String createIndexValue(Map<String, String> row, String[] nameIndexs) {
+ String[] indexValues = new String[nameIndexs.length];
+ for (int i = 0; i < nameIndexs.length; i++) {
+ indexValues[i] = row.get(nameIndexs[i]);
+ }
+ if (indexValues != null && indexValues.length > 0) {
+ String indexValue = MapKeyUtil.createKey(indexValues);
+ return indexValue;
+ }
+ return null;
+ }
+
+ /**
+ * 把row 中非string的值转化成string
+ *
+ * @param row
+ * @return
+ */
+ protected Map<String, String> createRow(Map<String, Object> row) {
+ Map<String, String> cacheValues = new HashMap<String, String>();//一行数据
+ Iterator<Map.Entry<String, Object>> iterator = row.entrySet().iterator();
+ //把数据value从object转化成string
+ while (iterator.hasNext()) {
+ Map.Entry<String, Object> entry = iterator.next();
+ if (entry != null && entry.getValue() != null && entry.getKey() != null) {
+ cacheValues.put(entry.getKey(), entry.getValue().toString());
+ }
+ }
+ return cacheValues;
+ }
+
+ public static IntDataType INTDATATYPE = new IntDataType();
+
+ /**
+ * 把确定多索引值，和行号放入索引缓存
+ *
+ * @param name2RowIndexs
+ * @param rowIndex
+ * @param indexValue
+ */
+ protected void addValue2Index(Map<String, List<Integer>> name2RowIndexs, Integer rowIndex,
+ String indexValue) {
+
+ List<Integer> rowIndexs = name2RowIndexs.get(indexValue);
+ if (rowIndexs == null) {
+ rowIndexs = new ArrayList<>();
+ name2RowIndexs.put(indexValue, rowIndexs);
+ }
+ rowIndexs.add(rowIndex);
+
+ }
+
+ public boolean isUnique() {
+ return isUnique;
+ }
+
+ public void setUnique(boolean unique) {
+ isUnique = unique;
+ }
+
+ public List<String> getIndexs() {
+ return indexs;
+ }
+
+ public void setIndexs(List<String> indexs) {
+ this.indexs = indexs;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/IndexExecutor.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/IndexExecutor.java
new file mode 100644
index 0000000..ceb4f52
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/IndexExecutor.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.index;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.function.expression.Equals;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression;
+import org.apache.rocketmq.streams.script.ScriptComponent;
+import org.apache.rocketmq.streams.common.datatype.IntDataType;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+
+/**
+ * 执行索引的查询和构建。主要是完成表达式的解析，对于等值的表达式字段，如果有索引，根据索引查询，然后执行非等值部分的判断
+ */
+public class IndexExecutor {
+
+ private String expressionStr;//表达式
+
+ private boolean isSupport = false;//是否支持索引，比如表达式等值部分无所以，则不能走索引逻辑
+
+ private String namespace;
+
+ private String indexNameKey;//索引的名字，多个字段";"分隔
+
+ private List<String> msgNames;//
+
+ private Rule rule;//表达式会被编译成rule
+
+ private List<String> index; //标准化后的索引name
+
+ private Set<String> indexNames = new HashSet<>();
+
+ public IndexExecutor(String expressionStr, String namespace, List<String> index) {
+ this.expressionStr = expressionStr;
+ this.namespace = namespace;
+
+ List<String> allIndex = new ArrayList<>();
+ for (String indexName : index) {
+ String[] values = indexName.split(";");
+ List<String> indexList = new ArrayList<>();
+ for (String value : values) {
+ indexNames.add(value);
+ indexList.add(value);
+ }
+ Collections.sort(indexList);
+ String indexKey = MapKeyUtil.createKey(indexList);
+ allIndex.add(indexKey);
+ }
+ this.index = allIndex;
+ parse();
+ }
+
+ /**
+ * 解析表达式，找出等值字段和非等值字段 如果有索引走索引，否则走全表扫描
+ */
+ protected void parse() {
+ List<Expression> expressions = new ArrayList<>();
+ List<RelationExpression> relationExpressions = new ArrayList<>();
+ Expression expression = ExpressionBuilder.createOptimizationExpression(namespace, "tmp", expressionStr,
+ expressions, relationExpressions);
+
+ RelationExpression relationExpression = null;
+ if (RelationExpression.class.isInstance(expression)) {
+ relationExpression = (RelationExpression)expression;
+ if (!"and".equals(relationExpression.getRelation())) {
+ isSupport = false;
+ return;
+ }
+ }
+
+ this.isSupport = true;
+ List<Expression> indexExpressions = new ArrayList<>();
+ List<Expression> otherExpressions = new ArrayList<>();
+ if (relationExpression != null) {
+ Map<String, Expression> map = new HashMap<>();
+ for (Expression tmp : expressions) {
+ map.put(tmp.getConfigureName(), tmp);
+ }
+ for (Expression tmp : relationExpressions) {
+ map.put(tmp.getConfigureName(), tmp);
+ }
+ List<String> expressionNames = relationExpression.getValue();
+ relationExpression.setValue(new ArrayList<>());
+ for (String expressionName : expressionNames) {
+ Expression subExpression = map.get(expressionName);
+ if (subExpression != null && !RelationExpression.class.isInstance(subExpression)) {
+ indexExpressions.add(subExpression);
+ } else {
+ otherExpressions.add(subExpression);
+ relationExpression.getValue().add(subExpression.getConfigureName());
+ }
+ }
+
+ } else {
+ indexExpressions.add(expression);
+ }
+
+ List<String> fieldNames = new ArrayList<>();
+ Map<String, String> msgNames = new HashMap<>();
+
+ for (Expression expre : indexExpressions) {
+ if (RelationExpression.class.isInstance(expre)) {
+ continue;
+ }
+ String indexName = expre.getValue().toString();
+ if (Equals.isEqualFunction(expre.getFunctionName()) && indexNames.contains(indexName)) {
+
+ fieldNames.add(indexName);
+ msgNames.put(indexName, expre.getVarName());
+ }
+ }
+ Collections.sort(fieldNames);
+ indexNameKey = MapKeyUtil.createKey(fieldNames);
+ if (!this.index.contains(indexNameKey)) {
+ this.isSupport = false;
+ return;
+ }
+ this.msgNames = createMsgNames(fieldNames, msgNames);
+ if (otherExpressions.size() == 0) {
+ return;
+ }
+ Rule rule = null;
+ if (relationExpression == null) {
+ rule = ExpressionBuilder.createRule(namespace, "tmpRule", expression);
+ } else {
+ rule = ExpressionBuilder.createRule(namespace, "tmpRule", expression, expressions, relationExpressions);
+ }
+
+ this.rule = rule;
+
+ }
+
+ /**
+ * 创建索引字段的索引值
+ *
+ * @param fieldNames
+ * @param msgNames
+ * @return
+ */
+ protected List<String> createMsgNames(List<String> fieldNames, Map<String, String> msgNames) {
+ List<String> msgNameList = new ArrayList<>();
+ for (String fieldName : fieldNames) {
+ msgNameList.add(msgNames.get(fieldName));
+ }
+ return msgNameList;
+ }
+
+ public boolean isSupport() {
+ return isSupport;
+ }
+
+ private static IntDataType intDataType = new IntDataType();
+
+ public List<Map<String, Object>> match(JSONObject msg, AbstractDim nameList, boolean needAll) {
+ return match(msg, nameList, needAll);
+ }
+
+ public List<Map<String, Object>> match(JSONObject msg, AbstractDim nameList, boolean needAll, String script) {
+ List<Map<String, Object>> rows = new ArrayList<>();
+ String msgValue = createValue(msg);
+ List<Integer> rowIds = nameList.findRowIdByIndex(indexNameKey, msgValue);
+ if (rowIds == null) {
+ return null;
+ }
+ for (Integer rowId : rowIds) {
+ Map<String, Object> oldRow = nameList.getDataCache().getRow(rowId);
+ Map<String, Object> newRow = executeScript(oldRow, script);
+ if (rule == null) {
+ rows.add(newRow);
+ if (needAll == false) {
+ return rows;
+ }
+ continue;
+ }
+ Rule ruleTemplete = this.rule;
+ Rule rule = ruleTemplete.copy();
+ Map<String, Expression> expressionMap = new HashMap<>();
+ for (Expression expression : rule.getExpressionMap().values()) {
+ expressionMap.put(expression.getConfigureName(), expression);
+ if (RelationExpression.class.isInstance(expression)) {
+ continue;
+ }
+ Object object = expression.getValue();
+ if (object != null && DataTypeUtil.isString(object.getClass())) {
+ String fieldName = (String)object;
+ Object o = newRow.get(fieldName);
+ if (o != null) {
+ Expression e = expression.copy();
+ e.setValue(o.toString());
+ expressionMap.put(e.getConfigureName(), e);
+ }
+ }
+ }
+ rule.setExpressionMap(expressionMap);
+ boolean matched = rule.execute(msg);
+ if (matched) {
+ rows.add(newRow);
+ if (needAll == false) {
+ return rows;
+ }
+ }
+ }
+ return rows;
+ }
+
+ protected Map<String, Object> executeScript(Map<String, Object> oldRow, String script) {
+ if (script == null) {
+ return oldRow;
+ }
+ ScriptComponent scriptComponent = ScriptComponent.getInstance();
+ JSONObject msg = new JSONObject();
+ msg.putAll(oldRow);
+ scriptComponent.getService().executeScript(msg, script);
+ return msg;
+ }
+
+ /**
+ * 按顺序创建msg的key
+ *
+ * @param msg
+ * @return
+ */
+ private String createValue(JSONObject msg) {
+ List<String> value = new ArrayList<>();
+ for (String msgName : msgNames) {
+ value.add(msg.getString(msgName));
+ }
+ return MapKeyUtil.createKey(value);
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AbstractIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AbstractIntelligenceCache.java
new file mode 100644
index 0000000..65b27f7
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AbstractIntelligenceCache.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.http.source.util.HttpUtil;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.rocketmq.streams.common.channel.sink.ISink;
+import org.apache.rocketmq.streams.common.utils.NumberUtils;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.dboperator.IDBDriver;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+
+public abstract class AbstractIntelligenceCache extends BasedConfigurable implements
+ IAfterConfiguableRefreshListerner {
+
+ private static final Log LOG = LogFactory.getLog(AbstractIntelligenceCache.class);
+
+ public static final String TYPE = "intelligence";
+
+ protected static final int FILE_MAX_LINE = 50000;//每个文件最大行数
+
+ protected transient IntValueKV intValueKV = new IntValueKV(0) {
+ @Override
+ public Integer get(String key) {
+ return null;
+ }
+
+ @Override
+ public void put(String key, Integer value) {
+
+ }
+ };
+
+ protected String idFieldName = "id";//必须有id字段
+
+ protected int batchSize = 3000;
+
+ @ENVDependence
+ protected Long pollingTimeMintue = 30L;
+
+ protected String datasourceName;//情报对应的存储
+
+ protected transient IDBDriver outputDataSource;
+
+ protected static ExecutorService executorService;
+
+ protected transient ScheduledExecutorService scheduledExecutorService;
+
+ public AbstractIntelligenceCache() {
+ setType(TYPE);
+ executorService = new ThreadPoolExecutor(20, 20,
+ 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(1000));
+ scheduledExecutorService = new ScheduledThreadPoolExecutor(3);
+ }
+
+ public IntValueKV startLoadData(String sql, IDBDriver resource) {
+ try {
+ String statisticalSQL = sql;
+ int startIndex = sql.toLowerCase().indexOf("from");
+ statisticalSQL = "select count(1) as c, min(" + idFieldName + ") as min, max(" + idFieldName + ") as max "
+ + sql.substring(startIndex);
+ List<Map<String, Object>> rows = resource.queryForList(statisticalSQL);
+ Map<String, Object> row = rows.get(0);
+ int count = Integer.valueOf(row.get("c").toString());
+ IntValueKV intValueKV = new IntValueKV(count);
+ //int maxBatch=count/maxSyncCount;//每1w条数据，一个并发。如果数据量比较大，为了提高性能，并行执行
+ if (count == 0) {
+ return new IntValueKV(0) {
+ @Override
+ public Integer get(String key) {
+ return null;
+ }
+
+ @Override
+ public void put(String key, Integer value) {
+
+ }
+ };
+ }
+ long min = Long.valueOf(row.get("min").toString());
+ long max = Long.valueOf(row.get("max").toString());
+ int maxSyncCount = count / FILE_MAX_LINE + 1;
+ long step = (max - min + 1) / maxSyncCount;
+ CountDownLatch countDownLatch = new CountDownLatch(maxSyncCount + 1);
+ AtomicInteger finishedCount = new AtomicInteger(0);
+ String taskSQL = null;
+ if (sql.indexOf(" where ") != -1) {
+ taskSQL = sql + " and " + idFieldName + ">#{startIndex} and " + idFieldName + "<=#{endIndex} order by "
+ + idFieldName + " limit " + batchSize;
+ } else {
+ taskSQL = sql + " where " + idFieldName + ">#{startIndex} and " + idFieldName
+ + "<=#{endIndex} order by " + idFieldName + " limit " + batchSize;
+ }
+
+ int i = 0;
+ for (; i < maxSyncCount; i++) {
+ FetchDataTask fetchDataTask = new FetchDataTask(taskSQL, (min - 1) + step * i,
+ (min - 1) + step * (i + 1), countDownLatch, finishedCount, resource, i, intValueKV, this, count);
+ executorService.execute(fetchDataTask);
+ }
+ FetchDataTask fetchDataTask = new FetchDataTask(taskSQL, (min - 1) + step * i, (min - 1) + step * (i + 1),
+ countDownLatch, finishedCount, resource, i, intValueKV, this, count);
+ executorService.execute(fetchDataTask);
+
+ countDownLatch.await();
+
+ LOG.info(getClass().getSimpleName() + " load data finish, load data line size is " + intValueKV.getSize());
+ return intValueKV;
+ } catch (Exception e) {
+ LOG.error("failed loading intelligence data!", e);
+ return new IntValueKV(0) {
+ @Override
+ public Integer get(String key) {
+ return null;
+ }
+
+ @Override
+ public void put(String key, Integer value) {
+
+ }
+ };
+ }
+ }
+
+ protected transient AtomicBoolean hasInit = new AtomicBoolean(false);
+
+ @Override
+ public void doProcessAfterRefreshConfigurable(IConfigurableService configurableService) {
+ this.outputDataSource = configurableService.queryConfigurable(ISink.TYPE, datasourceName);
+ }
+
+ public void startIntelligence() {
+ boolean success = dbInit();
+ if (success) {
+ startIntelligenceInner();
+ } else {
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ boolean success = false;
+ while (!success) {
+ success = dbInit();
+ try {
+ Thread.sleep(60 * 1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ startIntelligenceInner();
+ }
+ });
+ thread.start();
+ }
+ }
+
+ public void startIntelligenceInner() {
+ String sql = getSQL();
+ if (hasInit.compareAndSet(false, true)) {
+ this.intValueKV = startLoadData(sql, outputDataSource);
+ scheduledExecutorService.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ intValueKV = startLoadData(sql, outputDataSource);
+ }
+ }, pollingTimeMintue, pollingTimeMintue, TimeUnit.MINUTES);
+ }
+ }
+
+ public abstract Map<String, Object> getRow(String key);
+
+ /**
+ * 查询情报需要的sql
+ *
+ * @return
+ */
+ protected abstract String getSQL();
+
+ /**
+ * 情报中的 情报字段名
+ *
+ * @return
+ */
+ public abstract String getKeyName();
+
+ /**
+ * 情报对应的表名
+ *
+ * @return
+ */
+ public abstract String getTableName();
+
+ protected class FetchDataTask implements Runnable {
+ IntValueKV intValueKV;
+ long startIndex;
+ long endIndex;
+ String sql;
+ CountDownLatch countDownLatch;
+ int index;
+ IDBDriver resource;
+ AtomicInteger finishedCount;//完成了多少条
+ AbstractIntelligenceCache cache;
+ int totalSize;//一共有多少条数据
+
+ public FetchDataTask(String sql, long startIndex, long endIndex, CountDownLatch countDownLatch,
+ AtomicInteger finishedCount, IDBDriver resource, int i, IntValueKV intValueKV, AbstractIntelligenceCache cache, int totalSize) {
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ this.countDownLatch = countDownLatch;
+ this.sql = sql;
+ this.finishedCount = finishedCount;
+ this.resource = resource;
+ this.index = i;
+ this.intValueKV = intValueKV;
+ this.cache = cache;
+ this.totalSize = totalSize;
+ }
+
+ @Override
+ public void run() {
+ long currentIndex = startIndex;
+ JSONObject msg = new JSONObject();
+ msg.put("endIndex", endIndex);
+ while (true) {
+ try {
+
+ msg.put("startIndex", currentIndex);
+
+ String sql = SQLUtil.parseIbatisSQL(msg, this.sql);
+ List<Map<String, Object>> rows = resource.queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ break;
+ }
+ currentIndex = Long.valueOf(rows.get(rows.size() - 1).get(idFieldName).toString());
+
+ int size = rows.size();
+ int count = finishedCount.addAndGet(size);
+ double progress = (double)count / (double)totalSize;
+ progress = progress * 100;
+ System.out.println(cache.getClass().getSimpleName() + ", finished count is " + count + " the total count is " + totalSize + ", the progress is " + String.format("%.2f", progress) + "%");
+ if (size < batchSize) {
+ if (size > 0) {
+ doProccRows(intValueKV, rows, index);
+ }
+ break;
+ }
+ doProccRows(intValueKV, rows, index);
+ } catch (Exception e) {
+ throw new RuntimeException("put data error ", e);
+ }
+ }
+
+ countDownLatch.countDown();
+ }
+ }
+
+ public boolean dbInit() {
+ String uri = "/api/rest/rds/getDataSource";
+ String httpPrefix = "http";
+ String httpProtocol = "http://";
+ try {
+ String dbEndpoint = ComponentCreator.getProperties().getProperty(
+ ConfigureFileKey.INTELLIGENCE_TIP_DB_ENDPOINT);
+ if (StringUtils.isNotBlank(dbEndpoint)) {
+ if (!dbEndpoint.startsWith(httpPrefix)) {
+ dbEndpoint = httpProtocol + dbEndpoint;
+ }
+ String content = HttpUtil.getContent(dbEndpoint + uri);
+ if (StringUtils.isNotBlank(content)) {
+ JSONObject obj = JSON.parseObject(content);
+ String statusCode = "code";
+ int successCode = 200;
+ if (obj.getInteger(statusCode) == successCode) {
+ JSONObject data = obj.getJSONObject("data");
+ JSONObject dbInfo = data.getJSONObject("dBInfo");
+ if (dbInfo != null) {
+ String dbUrl = "jdbc:mysql://" + dbInfo.getString("dbConnection") + ":" + dbInfo.getInteger(
+ "port") + "/" + dbInfo.getString("dBName");
+ String dbUserName = dbInfo.getString("userName");
+ String dbPassword = dbInfo.getString("passWord");
+ JDBCDriver dataSource = (JDBCDriver)this.outputDataSource;
+ dataSource.setUrl(dbUrl);
+ dataSource.setPassword(dbPassword);
+ dataSource.setUserName(dbUserName);
+ dataSource.setHasInit(false);
+ dataSource.init();
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ /**
+ * 把存储0/1字符串的值，转化成bit
+ *
+ * @param values
+ * @return
+ */
+ protected int createInt(List<String> values) {
+ return NumberUtils.createBitMapInt(values);
+ }
+
+ /**
+ * 获取某位的值，如果是1，返回字符串1，否则返回null
+ *
+ * @param num
+ * @param i
+ * @return
+ */
+ protected String getNumBitValue(int num, int i) {
+ boolean exist = NumberUtils.getNumFromBitMapInt(num, i);
+ if (exist) {
+ return "1";
+ }
+ return null;
+ }
+
+ protected abstract void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index);
+
+ public String getIdFieldName() {
+ return idFieldName;
+ }
+
+ public void setIdFieldName(String idFieldName) {
+ this.idFieldName = idFieldName;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public Long getPollingTimeMintue() {
+ return pollingTimeMintue;
+ }
+
+ public void setPollingTimeMintue(Long pollingTimeMintue) {
+ this.pollingTimeMintue = pollingTimeMintue;
+ }
+
+ public String getDatasourceName() {
+ return datasourceName;
+ }
+
+ public void setDatasourceName(String datasourceName) {
+ this.datasourceName = datasourceName;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AccountIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AccountIntelligenceCache.java
new file mode 100644
index 0000000..9466f3e
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AccountIntelligenceCache.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * table: ads_yunsec_abnormal_account
+ */
+public class AccountIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+
+ private static final Log LOG = LogFactory.getLog(AccountIntelligenceCache.class);
+
+ /**
+ * 情报域名
+ */
+ protected transient String keyName = "account";
+
+ @Override
+ public Map<String, Object> getRow(String account) {
+ Integer value = intValueKV.get(account);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<String, Object>() {{
+ put("account", account);
+ }};
+ return row;
+ }
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id, `account` FROM `ads_yunsec_abnormal_account`";
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_abnormal_account";
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ rows.forEach(row -> {
+ String account = (String)row.get(keyName);
+ if (account != null) {
+ intValueKV.put(account, 1);
+ }
+ });
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/DomainIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/DomainIntelligenceCache.java
new file mode 100644
index 0000000..72a88bf
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/DomainIntelligenceCache.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DomainIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+ private static final Log LOG = LogFactory.getLog(DomainIntelligenceCache.class);
+ protected transient String keyName = "domain";
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id, `domain` , `is_malicious_source` , `is_phishing` , `is_c2` , `is_mining_pool` FROM `ads_yunsec_ti_domain_all_df` where curdate() < date_add(modify_time, interval expire_time day) ";
+ }
+
+ @Override
+ public Map<String, Object> getRow(String ip) {
+ Integer value = intValueKV.get(ip);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<>();
+ row.put("is_malicious_source", getNumBitValue(value, 0));
+ row.put("is_phishing", getNumBitValue(value, 1));
+ row.put("is_c2", getNumBitValue(value, 2));
+ row.put("is_mining_pool", getNumBitValue(value, 3));
+ return row;
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_ti_domain_all_df";
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ for (Map<String, Object> row : rows) {
+ String ip = (String)row.get(keyName);
+ if (ip == null) {
+ LOG.warn("load Intelligence exception ,the ip is null");
+ continue;
+ }
+ List<String> values = new ArrayList<>();
+ values.add((String)row.get("is_malicious_source"));
+ values.add((String)row.get("is_phishing"));
+ values.add((String)row.get("is_c2"));
+ values.add((String)row.get("is_mining_pool"));
+ int value = createInt(values);
+ synchronized (this) {
+ intValueKV.put(ip, value);
+ }
+
+ }
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/IPIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/IPIntelligenceCache.java
new file mode 100644
index 0000000..6382cd2
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/IPIntelligenceCache.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.dboperator.IDBDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+
+public class IPIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+ private static final Log LOG = LogFactory.getLog(IPIntelligenceCache.class);
+ protected transient String keyName = "ip";
+
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id,ip, `is_web_attack` , `is_tor` , `is_proxy` , `is_nat` , `is_mining_pool` , `is_c2` , "
+ + "`is_malicious_source` , `is_3rd` , `is_idc` , `is_malicious_login` FROM `ads_yunsec_ti_ip_all_df` where curdate() < date_add(modify_time, interval expire_time day) ";
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_ti_ip_all_df";
+ }
+
+ @Override
+ public Map<String, Object> getRow(String ip) {
+
+ Integer value = intValueKV.get(ip);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<>();
+
+ row.put("is_web_attack", getNumBitValue(value, 0));
+ row.put("is_tor", getNumBitValue(value, 1));
+ row.put("is_proxy", getNumBitValue(value, 2));
+ row.put("is_nat", getNumBitValue(value, 3));
+ row.put("is_mining_pool", getNumBitValue(value, 4));
+ row.put("is_c2", getNumBitValue(value, 5));
+ row.put("is_malicious_source", getNumBitValue(value, 6));
+ row.put("is_3rd", getNumBitValue(value, 7));
+ row.put("is_idc", getNumBitValue(value, 8));
+ row.put("is_malicious_login", getNumBitValue(value, 9));
+ return row;
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ for (Map<String, Object> row : rows) {
+ String ip = (String)row.get(keyName);
+ if (ip == null) {
+ LOG.warn("load Intelligence exception ,the ip is null");
+ continue;
+ }
+ List<String> values = new ArrayList<>();
+ values.add((String)row.get("is_web_attack"));
+ values.add((String)row.get("is_tor"));
+ values.add((String)row.get("is_proxy"));
+ values.add((String)row.get("is_nat"));
+ values.add((String)row.get("is_mining_pool"));
+ values.add((String)row.get("is_c2"));
+ values.add((String)row.get("is_malicious_source"));
+ values.add((String)row.get("is_3rd"));
+ values.add((String)row.get("is_idc"));
+ values.add((String)row.get("is_malicious_login"));
+ int value = createInt(values);
+ synchronized (this) {
+ intValueKV.put(ip, value);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ ComponentCreator.setProperties(
+ "/Users/yuanxiaodong/Documents/workdir/档案/阿里安全/专有云/2020/dipper-siem/siem.properties");
+ IPIntelligenceCache ipIntelligenceCache = new IPIntelligenceCache();
+ IDBDriver outputDataSource = DriverBuilder.createDriver();
+ ipIntelligenceCache.startLoadData(ipIntelligenceCache.getSQL(), outputDataSource);
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/URLIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/URLIntelligenceCache.java
new file mode 100644
index 0000000..8910469
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/URLIntelligenceCache.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class URLIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+
+ private static final Log LOG = LogFactory.getLog(URLIntelligenceCache.class);
+
+ protected transient String keyName = "url";
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id, url, `is_malicious_source` FROM `ads_yunsec_ti_url_all_df` where curdate() < date_add(modify_time, interval expire_time day) ";
+ }
+
+ @Override
+ public Map<String, Object> getRow(String ip) {
+ Integer value = intValueKV.get(ip);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<>();
+
+ row.put("is_malicious_source", getNumBitValue(value, 0));
+ return row;
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_ti_url_all_df";
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ for (Map<String, Object> row : rows) {
+ String ip = (String)row.get(keyName);
+ if (ip == null) {
+ LOG.warn("load Intelligence exception ,the ip is null");
+ continue;
+ }
+ List<String> values = new ArrayList<>();
+ values.add((String)row.get("is_malicious_source"));
+ int value = createInt(values);
+ synchronized (this) {
+ intValueKV.put(ip, value);
+ }
+
+ }
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/AbstractDim.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/AbstractDim.java
new file mode 100644
index 0000000..c51b0e6
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/AbstractDim.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.model;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.dim.index.IndexExecutor;
+import org.apache.rocketmq.streams.dim.index.DimIndex;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression;
+import org.apache.rocketmq.streams.common.cache.softreference.ICache;
+import org.apache.rocketmq.streams.common.cache.softreference.impl.SoftReferenceCache;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.cache.CompressTable;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * 这个结构代表一张表 存放表的全部数据和索引
+ */
+public abstract class AbstractDim extends BasedConfigurable {
+
+ private static final Log LOG = LogFactory.getLog(AbstractDim.class);
+
+ public static final String TYPE = "nameList";
+
+ /**
+ * 同步数据的事件间隔，单位是分钟
+ */
+ protected Long pollingTimeMintue = 60L;
+
+ /**
+ * 支持多组索引，如果一个索引是组合索引，需要拼接成一个string，用;分割 建立索引后，会创建索引的数据结构，类似Map<String,List<RowId>，可以快速定位，无索引会全表扫描，不建议使用 如有两组索引：1.name 2. ip;address
+ */
+ protected List<String> indexs = new ArrayList<>();
+
+ /**
+ * 把表数据转化成二进制存储在CompressTable中
+ */
+ protected transient volatile CompressTable dataCache;
+
+ /**
+ * 建立名单的时候，可以指定多组索引，索引的值当作key，row在datacache的index当作value，可以快速匹配索引对应的row key：索引的值 value：row在dataCache的index当作value，可以快速匹配索引对应的row
+ */
+ protected transient DimIndex nameListIndex;
+
+ //是否是唯一索引，唯一索引会用IntValueKV存储，有更高的压缩率
+ protected boolean isUniqueIndex = false;
+
+ //定时加载表数据到内存
+ protected transient ScheduledExecutorService executorService;
+
+ public AbstractDim() {
+ this.setType(TYPE);
+ }
+
+ //protected String index;//只是做标记，为了是简化indexs的赋值
+
+ public String addIndex(String... fieldNames) {
+ return addIndex(this.indexs, fieldNames);
+ }
+
+ @Override
+ protected boolean initConfigurable() {
+ boolean success = super.initConfigurable();
+ loadNameList();
+ executorService = new ScheduledThreadPoolExecutor(3);
+ executorService.scheduleWithFixedDelay(new Runnable() {
+
+ @Override
+ public void run() {
+ loadNameList();
+ }
+ }, pollingTimeMintue, pollingTimeMintue, TimeUnit.MINUTES);
+ return success;
+ }
+
+ /**
+ * 加载维表数据 创建索引
+ */
+ protected void loadNameList() {
+ try {
+ LOG.info(getConfigureName() + " begin polling data");
+ //全表数据
+ CompressTable dataCacheVar = loadData();
+ this.dataCache = dataCacheVar;
+ this.nameListIndex = buildIndex(dataCacheVar);
+ } catch (Exception e) {
+ LOG.error("Load configurables error:" + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * 给维表生成索引数据结构
+ *
+ * @param dataCacheVar 维表
+ * @return
+ */
+ protected DimIndex buildIndex(CompressTable dataCacheVar) {
+ DimIndex dimIndex = new DimIndex(this.indexs);
+ dimIndex.setUnique(isUniqueIndex);
+ dimIndex.buildIndex(dataCacheVar);
+ return dimIndex;
+ }
+
+ /**
+ * 根据索引名和索引值查询匹配的行号
+ *
+ * @param indexName
+ * @param indexValue
+ * @return
+ */
+ public List<Integer> findRowIdByIndex(String indexName, String indexValue) {
+ return nameListIndex == null ? Collections.emptyList() : nameListIndex.getRowIds(indexName, indexValue);
+ }
+
+ /**
+ * 软引用缓存，最大可能保存索引执行器，避免频繁创建，带来额外开销 同时会保护内存不被写爆，当内存不足时自动回收内存
+ */
+ private static ICache<String, IndexExecutor> cache = new SoftReferenceCache<>();
+
+ /**
+ * 先找索引，如果有索引，通过索引匹配。如果没有，全表扫表.
+ *
+ * @param expressionStr 表达式
+ * @param msg 消息
+ * @return 只返回匹配的第一行
+ */
+ public Map<String, Object> matchExpression(String expressionStr, JSONObject msg) {
+ List<Map<String, Object>> rows = matchExpression(expressionStr, msg, true, null);
+ if (rows != null && rows.size() > 0) {
+ return rows.get(0);
+ }
+ return null;
+ }
+
+ /**
+ * 先找索引，如果有索引，通过索引匹配。如果没有，全表扫表
+ *
+ * @param expressionStr 表达式
+ * @param msg 消息
+ * @return 返回全部匹配的行
+ */
+ public List<Map<String, Object>> matchExpression(String expressionStr, JSONObject msg, boolean needAll, String script) {
+ IndexExecutor indexNamelistExecutor = cache.get(expressionStr);
+ if (indexNamelistExecutor == null) {
+ indexNamelistExecutor = new IndexExecutor(expressionStr, getNameSpace(), this.indexs);
+ cache.put(expressionStr, indexNamelistExecutor);
+ }
+ if (indexNamelistExecutor.isSupport()) {
+ return indexNamelistExecutor.match(msg, this, needAll, script);
+ } else {
+ return matchExpressionByLoop(expressionStr, msg, needAll);
+ }
+ }
+
+ /**
+ * 全表扫描，做表达式匹配，返回全部匹配结果
+ *
+ * @param expressionStr
+ * @param msg
+ * @param needAll
+ * @return
+ */
+ protected List<Map<String, Object>> matchExpressionByLoop(String expressionStr, JSONObject msg, boolean needAll) {
+ CompressTable dataCache = this.dataCache;
+ List<Map<String, Object>> rows = matchExpressionByLoop(dataCache.newIterator(), expressionStr, msg, needAll);
+ return rows;
+ }
+
+ /**
+ * 全表扫描，做表达式匹配，返回全部匹配结果。join中有使用
+ *
+ * @param expressionStr
+ * @param msg
+ * @param needAll
+ * @return
+ */
+ public static List<Map<String, Object>> matchExpressionByLoop(Iterator<Map<String, Object>> it, String expressionStr, JSONObject msg, boolean needAll) {
+ List<Map<String, Object>> rows = new ArrayList<>();
+ while (it.hasNext()) {
+ Map<String, Object> values = it.next();
+ Rule ruleTemplete = ExpressionBuilder.createRule("tmp", "tmpRule", expressionStr);
+ Rule rule = ruleTemplete.copy();
+ Map<String, Expression> expressionMap = new HashMap<>();
+ for (Expression expression : rule.getExpressionMap().values()) {
+ expressionMap.put(expression.getConfigureName(), expression);
+ if (RelationExpression.class.isInstance(expression)) {
+ continue;
+ }
+ Object object = expression.getValue();
+ if (object != null && DataTypeUtil.isString(object.getClass())) {
+ String fieldName = (String)object;
+ Object value = values.get(fieldName);
+ if (value != null) {
+ Expression e = expression.copy();
+ e.setValue(value.toString());
+ expressionMap.put(e.getConfigureName(), e);
+ }
+ }
+ }
+ rule.setExpressionMap(expressionMap);
+ boolean matched = rule.execute(msg);
+ if (matched) {
+ rows.add(values);
+ if (needAll == false) {
+ return rows;
+ }
+ }
+ }
+ return rows;
+ }
+
+ protected abstract CompressTable loadData();
+
+ @Override
+ public void destroy() {
+ super.destroy();
+ executorService.shutdown();
+ }
+
+ /**
+ * 设置索引
+ *
+ * @param indexs 字段名称，多个字段";"分隔
+ */
+ public void setIndex(String indexs) {
+ if (StringUtil.isEmpty(indexs)) {
+ return;
+ }
+ List<String> tmp = new ArrayList<>();
+ String[] values = indexs.split(";");
+ this.addIndex(tmp, values);
+ this.indexs = tmp;
+ }
+
+ /**
+ * 建议指定索引，会基于索引建立map，对于等值的判断，可以快速匹配
+ *
+ * @param fieldNames
+ */
+ private String addIndex(List<String> indexs, String... fieldNames) {
+ if (fieldNames == null) {
+ return null;
+ }
+ Arrays.sort(fieldNames);
+ String index = MapKeyUtil.createKey(fieldNames);
+ if (StringUtil.isNotEmpty(index)) {
+ indexs.add(index);
+ }
+ return index;
+ }
+
+ public Long getPollingTimeMintue() {
+ return pollingTimeMintue;
+ }
+
+ public void setPollingTimeMintue(Long pollingTimeMintue) {
+ this.pollingTimeMintue = pollingTimeMintue;
+ }
+
+ public List<String> getIndexs() {
+ return indexs;
+ }
+
+ public void setIndexs(List<String> indexs) {
+ this.indexs = indexs;
+ }
+
+ public CompressTable getDataCache() {
+ return dataCache;
+ }
+
+ public boolean isUniqueIndex() {
+ return isUniqueIndex;
+ }
+
+ public void setUniqueIndex(boolean uniqueIndex) {
+ isUniqueIndex = uniqueIndex;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/BooleanFieldDBDim.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/BooleanFieldDBDim.java
new file mode 100644
index 0000000..80dd2d2
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/BooleanFieldDBDim.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.model;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.metadata.MetaData;
+import org.apache.rocketmq.streams.common.metadata.MetaDataField;
+
+/**
+ * 类似情报表，除了一个核心比对字段和主键，其他都是boolean类型的/或者是只有0/1两个值的字符串和int。可以用这个存储结构
+ */
+public class BooleanFieldDBDim extends DBDim {
+
+ private static final Log LOG = LogFactory.getLog(BooleanFieldDBDim.class);
+
+ /**
+ * 类似情报表，除了一个核心比对字段和主键，其他都是boolean类型的。可以用这个存储结构
+ *
+ * @param metaData
+ * @param notBooleanFieldName
+ * @return
+ */
+ public static boolean support(MetaData metaData, String notBooleanFieldName) {
+ List<MetaDataField> metaDataFields = metaData.getMetaDataFields();
+ for (MetaDataField metaDataField : metaDataFields) {
+ if (metaDataField.getIsPrimary()) {
+ continue;
+ }
+ if (metaDataField.getFieldName().equals(notBooleanFieldName)) {
+ continue;
+ }
+ if (!metaDataField.getDataType().matchClass(Boolean.class)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/DBDim.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/DBDim.java
new file mode 100644
index 0000000..6180820
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/DBDim.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.model;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.cache.CompressTable;
+import org.apache.rocketmq.streams.common.utils.IPUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+
+public class DBDim extends AbstractDim {
+
+ private static final Log LOG = LogFactory.getLog(DBDim.class);
+
+ private String jdbcdriver = "com.mysql.jdbc.Driver";
+
+ @ENVDependence
+ private String url;
+
+ @ENVDependence
+ private String userName;
+
+ @ENVDependence
+ private String password;
+
+ private String sql;//sql 会被定时执行
+
+ private static transient AtomicInteger nameCreator = new AtomicInteger(0);
+
+ /**
+ * 是否支持批量查找
+ */
+ protected transient Boolean supportBatch = false;
+
+ public DBDim() {
+ this.setConfigureName(MapKeyUtil.createKey(IPUtil.getLocalIdentification(), System.currentTimeMillis() + "",
+ nameCreator.incrementAndGet() + ""));
+ this.setType(TYPE);
+ }
+
+ @Override
+ protected CompressTable loadData() {
+ List<Map<String, Object>> rows = executeQuery();
+ CompressTable tableCompress = new CompressTable();
+ for (Map<String, Object> row : rows) {
+ tableCompress.addRow(row);
+ }
+ return tableCompress;
+ }
+
+ protected List<Map<String, Object>> executeQuery() {
+ JDBCDriver resource = createResouce();
+ try {
+ List<Map<String, Object>> result = resource.queryForList(sql);
+ ;
+ LOG.info("load configurable's count is " + result.size());
+ return result;
+ } finally {
+ if (resource != null) {
+ resource.destroy();
+ }
+ }
+
+ }
+
+ protected JDBCDriver createResouce() {
+ return DriverBuilder.createDriver(jdbcdriver, url, userName, password);
+ }
+
+ public void setJdbcdriver(String jdbcdriver) {
+ this.jdbcdriver = jdbcdriver;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public void setSql(String sql) {
+ this.sql = sql;
+ }
+
+ public String getJdbcdriver() {
+ return jdbcdriver;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getSql() {
+ return sql;
+ }
+
+ public Boolean getSupportBatch() {
+ return supportBatch;
+ }
+
+ public void setSupportBatch(Boolean supportBatch) {
+ this.supportBatch = supportBatch;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/IDimService.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/IDimService.java
new file mode 100644
index 0000000..c5c4a2e
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/IDimService.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.service;
+
+import java.util.List;
+import java.util.Map;
+
+public interface IDimService {
+
+ /**
+ * 做维表join，关系通过表达式表示，返回所有匹配的行。因为msg没有key，表达式中，以下标表示key，如0，1，2。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式（0,functionName,filedName)&（1,functionName,filedName)|（2,functionName,filedName)
+ * @param msgs 流数据
+ * @return 符合匹配条件的所有行
+ */
+ Map<String, Object> match(String dimName, String expressionStr, Object... msgs);
+
+ /**
+ * 做维表join，关系通过表达式表示，返回所有匹配的行。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式,varName是msg中的key名称（varName,functionName,filedName)&（varName,functionName,filedName)|（varName,functionName,filedName)
+ * @param msgs 流数据
+ * @return 符合匹配条件的所有行
+ */
+ List<Map<String, Object>> matchSupportMultiRow(String dimName, String expressionStr, Map<String, Object> msgs);
+
+ /**
+ * 做维表join，关系通过表达式表示，返回匹配的一行数据，如果有多行匹配，只返回第一行。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式,varName是msg中的key名称（varName,functionName,filedName)&（varName,functionName,filedName)|（varName,functionName,filedName)
+ * @param msgs 流数据
+ * @return 返回匹配的一行数据，如果有多行匹配，只返回第一行。
+ */
+ Map<String, Object> match(String dimName, String expressionStr, Map<String, Object> msgs);
+
+ /**
+ * 做维表join，关系通过表达式表示，返回匹配的全部数据。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式,varName是msg中的key名称（varName,functionName,filedName)&（varName,functionName,filedName)|（varName,functionName,filedName)
+ * @param msgs 流数据
+ * @param script 对维表字段做处理的函数,在执行表达式前需要先对维表字段做处理，如fiedlName=trim(fieldName)
+ * @return 返回匹配的一行数据，如果有多行匹配，只返回第一行。
+ */
+ List<Map<String, Object>> matchSupportMultiRow(String dimName,
+ String expressionStr, Map<String, Object> msgs, String script);
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/impl/DimServiceImpl.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/impl/DimServiceImpl.java
new file mode 100644
index 0000000..20f22ae
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/impl/DimServiceImpl.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.service.impl;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.dim.service.IDimService;
+
+public class DimServiceImpl implements IDimService {
+ protected ConfigurableComponent configurableComponent;
+
+ public DimServiceImpl(ConfigurableComponent configurableComponent) {
+ this.configurableComponent = configurableComponent;
+ }
+
+ /**
+ * 传入要比对的字段，进行规则匹配。字段和名单的比对逻辑，写在规则中
+ *
+ * @param msgs 字段名默认为数组的索引，如1，2，3
+ * @return
+ */
+ @Override
+ public Map<String, Object> match(String dimName, String expressionStr, Object... msgs) {
+ if (msgs == null || msgs.length == 0) {
+ return null;
+ }
+ int i = 0;
+ JSONObject jsonObject = new JSONObject();
+ for (Object o : msgs) {
+ jsonObject.put(i + "", o);
+ i++;
+ }
+ return match(dimName, expressionStr, jsonObject);
+ }
+
+ @Override
+ public List<Map<String, Object>> matchSupportMultiRow(String dimName, String expressionStr, Map<String, Object> msgs) {
+ return matchSupportMultiRow(dimName, expressionStr, msgs, null);
+ }
+
+ @Override
+ public List<Map<String, Object>> matchSupportMultiRow(String dimName, String expressionStr, Map<String, Object> msgs, String script) {
+ JSONObject jsonObject = createJsonable(msgs);
+ AbstractDim nameList = configurableComponent.queryConfigurable(AbstractDim.TYPE, dimName);
+ if (nameList != null) {
+ return nameList.matchExpression(expressionStr, jsonObject, true, script);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Map<String, Object> match(String nameListName, String expressionStr, Map<String, Object> parameters) {
+ JSONObject jsonObject = createJsonable(parameters);
+ AbstractDim nameList = configurableComponent.queryConfigurable(AbstractDim.TYPE, nameListName);
+ if (nameList != null) {
+ return nameList.matchExpression(expressionStr, jsonObject);
+ } else {
+ return null;
+ }
+ }
+
+ private JSONObject createJsonable(Map<String, Object> parameters) {
+ JSONObject jsonObject = null;
+ if (parameters instanceof JSONObject) {
+ jsonObject = (JSONObject)parameters;
+ } else {
+ jsonObject.putAll(parameters);
+ }
+ return jsonObject;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/ConfigureLoaderTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ConfigureLoaderTest.java
new file mode 100644
index 0000000..3f43a2a
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ConfigureLoaderTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class ConfigureLoaderTest {
+
+ @Test
+ public void testLoadResource() throws IOException {
+ // BufferedReader br = new BufferedReader(new InputStreamReader(RuleEngineRunner.class.getClassLoader
+ // ().getResourceAsStream(".")));
+ // String line = br.readLine();
+ // while (line != null) {
+ // System.out.println(line);
+ // line = br.readLine();
+ // }
+ }
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/ExpressionExecutorTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ExpressionExecutorTest.java
new file mode 100644
index 0000000..380952f
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ExpressionExecutorTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.operator.expression.SimpleExpression;
+import org.apache.rocketmq.streams.filter.FilterComponent;
+import org.junit.Test;
+
+import java.io.File;
+
+public class ExpressionExecutorTest {
+ private static final String CREDIBLE_PROPERTIES = "credible" + File.separator + "credible.properties";
+ private FilterComponent filterComponent;
+
+ private String namespace = "yundun.credible.net.vistor";
+ private String ruleNameSpace = "credible.rule.net.vistor";
+ private String selectorName = "credible.selector.net.vistor";
+ private String selectorExpression = "(host_uuid,=,0a2153e2-e45c-403f-8d5f-d811f400c3fb)";
+ private String procWriteList = "credible.namelist.proc";
+ private String netWriteList = "credible.namelist.net.vistor";
+
+ private String ruleExpression =
+ "(proc_path,in_resouce," + namespace + "->" + procWriteList + ")&(inner_message,not_in_expression_resouce,'"
+ + namespace + "->" + netWriteList
+ + "->(visitor_ip,=,dest_ip)&(visitor_port,=,dest_port)&(proc_path,=,program_path)')";
+
+ // @Test
+ // public void parseExpression() {
+ // List<Expression> expressions = new ArrayList<>();
+ // List<RelationExpression> relationExpressions = new ArrayList<>();
+ // Expression expression = ExpressionBuilder.createExpression("namespace", ruleExpression,
+ // expressions,
+ // relationExpressions);
+ // }
+
+ public ExpressionExecutorTest() {
+ // FilterComponent filterComponent= new FilterComponent();
+ // filterComponent.init(CREDIBLE_PROPERTIES);
+ // filterComponent.start(null);
+ // this.filterComponent=filterComponent;
+ }
+
+ @Test
+ public void testExecutor() {
+ System.out.println("hello wolrd");
+ JSONObject msg = new JSONObject();
+ msg.put("ip", "1.1.1.1");
+ boolean match = ExpressionBuilder.executeExecute(new SimpleExpression("ip", "=", "1.1.1.1"), msg);
+ System.out.println(match);
+ }
+
+ @Test
+ public void testRelationExecutor() {
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("ip", "1.2.2.3");
+ jsonObject.put("uid", "1224");
+ jsonObject.put("vmip", "1.1.1.1");
+
+ boolean value =
+ ExpressionBuilder.executeExecute("namespace", "(ip,=,1.2.2.3)&((uid,=,12214)|(vmip,=,1.1.11.1))",
+ jsonObject);
+ System.out.println(value);
+ }
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/JsonParserTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/JsonParserTest.java
new file mode 100644
index 0000000..8f58a04
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/JsonParserTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import org.junit.Test;
+
+public class JsonParserTest {
+
+ @Test
+ public void testJson() {
+ String array =
+ "[{\"value\":\"groupname\",\"key\":\"group_name\"},{\"value\":\"username\",\"key\":\"user_name\"},"
+ + "{\"value\":\"seq\",\"key\":\"index\"},{\"value\":\"egroupid\",\"key\":\"egroup_id\"},"
+ + "{\"value\":\"filepath\",\"key\":\"file_path\"},{\"value\":\"groupid\",\"key\":\"group_id\"},"
+ + "{\"value\":\"pfilename\",\"key\":\"pfile_path\"},{\"value\":\"safe_mode\",\"key\":\"perm\"},"
+ + "{\"value\":\"cmdline\",\"key\":\"cmd_line\"}]";
+ String jsonStr =
+ "{\"className\":\"com.aliyun.filter.result.FieldReNameScript\",\"oldField2NewFiled\":" + array + "}";
+ JSONArray jsonObject = JSON.parseArray(array);
+ JSONObject js = JSON.parseObject(jsonStr);
+ System.out.println(js.toJSONString());
+ }
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/NameListFunctionTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/NameListFunctionTest.java
new file mode 100644
index 0000000..80d044c
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/NameListFunctionTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.junit.Test;
+
+public class NameListFunctionTest {
+
+ @Test
+ public void testNameList() {
+ AbstractDim nameList = create();
+ JSONObject msg = new JSONObject();
+ msg.put("ip", "47.105.77.144");
+ msg.put("vpcId", "1");
+ msg.put("now", "2019-07-18 17:33:29");
+ long start = System.currentTimeMillis();
+ }
+
+ @Test
+ public void testNameList2() {
+ AbstractDim nameList = createMapping();
+ JSONObject msg = new JSONObject();
+ msg.put("levelFile", "aegis-vul_record:level");
+ msg.put("levelValue", "high");
+ msg.put("now", "2019-07-18 17:33:29");
+ long start = System.currentTimeMillis();
+ }
+
+ private AbstractDim create() {
+ DBDim dbNameList = new DBDim();
+ dbNameList.setNameSpace("soc");
+ dbNameList.setConfigureName("isoc_field_mappings");
+ dbNameList.setUrl("");
+ dbNameList.setUserName("");
+ dbNameList.setPassword("");
+ dbNameList.setSql("SELECT * FROM `ecs_info` WHERE STATUS=1 LIMIT 1");
+ List<String> ipFieldNames = new ArrayList<>();
+ ipFieldNames.add("public_ips");
+ ipFieldNames.add("inner_ips");
+ ipFieldNames.add("eip");
+ ipFieldNames.add("private_ips");
+ dbNameList.init();
+ return dbNameList;
+ }
+
+ @Test
+ public void testNameListAllRow() {
+ AbstractDim nameList = createMapping();
+ JSONObject msg = new JSONObject();
+ msg.put("levelFile", "aegis-vul_record:level");
+ msg.put("levelValue", "high");
+ msg.put("now", "2019-07-18 17:33:29");
+ long start = System.currentTimeMillis();
+
+ }
+
+ private AbstractDim createMapping() {
+ DBDim dbNameList = new DBDim();
+ dbNameList.setNameSpace("soc");
+ dbNameList.setConfigureName("isoc_field_mappings");
+ dbNameList.setUrl("");
+ dbNameList.setUserName("");
+ dbNameList.setPassword("");
+ dbNameList.setSql("select * from ads_yunsec_ti_url_all_df limit 100000");
+ dbNameList.init();
+ return dbNameList;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/TableCompressTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/TableCompressTest.java
new file mode 100644
index 0000000..3c89fd6
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/TableCompressTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import org.junit.Test;
+
+public class TableCompressTest {
+
+ @Test
+ public void testNameList() throws InterruptedException {
+ }
+}
diff --git a/rocketmq-streams-lease/pom.xml b/rocketmq-streams-lease/pom.xml
new file mode 100755
index 0000000..a0ad067
--- /dev/null
+++ b/rocketmq-streams-lease/pom.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-lease</artifactId>
+ <name>ROCKETMQ STREAMS :: lease</name>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-db-operator</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/LeaseComponent.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/LeaseComponent.java
new file mode 100644
index 0000000..3a527a3
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/LeaseComponent.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */package org.apache.rocketmq.streams.lease;
+
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.component.ConfigureDescriptor;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+import org.apache.rocketmq.streams.lease.service.ILeaseStorage;
+import org.apache.rocketmq.streams.lease.service.impl.LeaseServiceImpl;
+import org.apache.rocketmq.streams.lease.service.impl.MockLeaseImpl;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.lease.service.storages.DBLeaseStorage;
+import org.apache.rocketmq.streams.serviceloader.ServiceLoaderComponent;
+
+/**
+ * 通过db实现租约和锁，可以更轻量级，减少其他中间件的依赖 使用主备场景，只有一个实例运行，当当前实例挂掉，在一定时间内，会被其他实例接手 也可以用于全局锁
+ *
+ * @date 1/9/19
+ */
+public class LeaseComponent extends AbstractComponent<ILeaseService> {
+
+ private static LeaseComponent leaseComponent = null;
+ private static final Log LOG = LogFactory.getLog(LeaseComponent.class);
+ private ILeaseService leaseService;
+
+ public LeaseComponent() {
+ initConfigurableServiceDescriptor();
+ addConfigureDescriptor(
+ new ConfigureDescriptor(CONNECT_TYPE, false, ConfigurableServcieType.DEFAULT_SERVICE_NAME));
+ }
+
+ public static LeaseComponent getInstance() {
+ if(leaseComponent==null){
+ synchronized (LeaseComponent.class){
+ if(leaseComponent==null){
+ leaseComponent =ComponentCreator.getComponent(null,LeaseComponent.class);
+ }
+ }
+ }
+ return leaseComponent;
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ @Override
+ public ILeaseService getService() {
+ return leaseService;
+ }
+
+ @Override
+ protected boolean startComponent(String namespace) {
+ return true;
+ }
+
+ @Override
+ protected boolean initProperties(Properties properties) {
+ String connectType = properties.getProperty(JDBC_URL);
+ if (StringUtil.isEmpty(connectType)) {
+ MockLeaseImpl mockLease = new MockLeaseImpl();
+ this.leaseService=mockLease;
+ return true;
+ }
+
+ LeaseServiceImpl leaseService= new LeaseServiceImpl();
+ String storageName=ComponentCreator.getProperties().getProperty(ConfigureFileKey.LEASE_STORAGE_NAME);
+ ILeaseStorage storasge=null;
+ if(StringUtil.isEmpty(storageName)){
+ String jdbc = properties.getProperty(AbstractComponent.JDBC_DRIVER);
+ String url = properties.getProperty(AbstractComponent.JDBC_URL);
+ String userName = properties.getProperty(AbstractComponent.JDBC_USERNAME);
+ String password = properties.getProperty(AbstractComponent.JDBC_PASSWORD);
+ storasge=new DBLeaseStorage(jdbc,url,userName,password);
+ }else {
+ storasge= (ILeaseStorage)ServiceLoaderComponent.getInstance(ILeaseStorage.class).loadService(storageName);
+ }
+ leaseService.setLeaseStorage(storasge);
+ this.leaseService=leaseService;
+ return true;
+ }
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/model/LeaseInfo.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/model/LeaseInfo.java
new file mode 100644
index 0000000..469a711
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/model/LeaseInfo.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.model;
+
+/**
+ * 租约对象，需要创建租约表。
+ */
+
+import java.io.Serializable;
+import java.util.Date;
+
+public class LeaseInfo implements Serializable {
+ private static final long serialVersionUID = 665608838255753618L;
+ private Long id;
+ private Date createTime;
+ private Date updateTime;
+ private String leaseName;//租约名称，多个进程共享一个租约，只要名称相同即可
+ private String leaseUserIp;//区分不同的租约实体，以前默认用ip，但一个机器多个进程的情况下，用ip会区分不开，后续会加上进程号
+ private Date leaseEndDate;//租约到期时间
+ private int status;//租约的有效状态
+ private long version;//版本，通过版本保证更新原子性
+
+ public LeaseInfo() {
+ }
+
+ /**
+ * 建表语句
+ *
+ * @return
+ */
+ public static String createTableSQL() {
+ return "CREATE TABLE IF NOT EXISTS `lease_info` (\n"
+ + " `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',\n"
+ + " `gmt_create` datetime NOT NULL COMMENT '创建时间',\n"
+ + " `gmt_modified` datetime NOT NULL COMMENT '修改时间',\n"
+ + " `lease_name` varchar(255) NOT NULL COMMENT '租约名称',\n"
+ + " `lease_user_ip` varchar(255) NOT NULL COMMENT '租者IP',\n"
+ + " `lease_end_time` varchar(255) NOT NULL COMMENT '租约到期时间',\n"
+ + " `status` int(11) NOT NULL DEFAULT '1' COMMENT '状态',\n"
+ + " `version` bigint(20) NOT NULL COMMENT '版本',\n"
+ + " `candidate_lease_ip` varchar(255) DEFAULT NULL COMMENT '候选租约ip',\n"
+ + " PRIMARY KEY (`id`),\n"
+ + " UNIQUE KEY `uk_name` (`lease_name`)\n"
+ + ") ENGINE=InnoDB AUTO_INCREMENT=8150 DEFAULT CHARSET=utf8 COMMENT='租约信息'\n"
+ + ";";
+ }
+
+ public Long getId() {
+ return this.id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public Date getCreateTime() {
+ return this.createTime;
+ }
+
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ public Date getUpdateTime() {
+ return this.updateTime;
+ }
+
+ public void setUpdateTime(Date updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ public String getLeaseUserIp() {
+ return this.leaseUserIp;
+ }
+
+ public void setLeaseUserIp(String leaseUserIp) {
+ this.leaseUserIp = leaseUserIp;
+ }
+
+ public Date getLeaseEndDate() {
+ return this.leaseEndDate;
+ }
+
+ public void setLeaseEndDate(Date leaseEndDate) {
+ this.leaseEndDate = leaseEndDate;
+ }
+
+ public int getStatus() {
+ return this.status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public String getLeaseName() {
+ return this.leaseName;
+ }
+
+ public void setLeaseName(String leaseName) {
+ this.leaseName = leaseName;
+ }
+
+ public long getVersion() {
+ return this.version;
+ }
+
+ public void setVersion(long version) {
+ this.version = version;
+ }
+
+}
+
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseGetCallback.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseGetCallback.java
new file mode 100644
index 0000000..ad63dde
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseGetCallback.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service;
+
+import java.util.Date;
+
+public interface ILeaseGetCallback {
+
+ /**
+ * 当成功获取租约时，回调接口
+ *
+ * @param nextLeaseDate 租约到期时间
+ */
+ void callback(Date nextLeaseDate);
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseService.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseService.java
new file mode 100644
index 0000000..7782c52
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseService.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service;
+
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+
+/**
+ * 通过db实现租约和锁，可以更轻量级，减少其他中间件的依赖 使用主备场景，只有一个实例运行，当当前实例挂掉，在一定时间内，会被其他实例接手 也可以用于全局锁
+ */
+public interface ILeaseService {
+
+ /**
+ * 默认锁定时间
+ */
+ static final int DEFALUT_LOCK_TIME = 60 * 5;
+
+ /**
+ * 检查某用户当前时间是否具有租约。这个方法是纯内存操作，无性能开销
+ *
+ * @return true，租约有效；false，租约无效
+ */
+ boolean hasLease(String name);
+
+ /**
+ * 申请租约，会启动一个线程，不停申请租约，直到申请成功。 申请成功后，每 租期/2 续约。 如果目前被其他租户获取租约，只有在对方租约失效，后才允许新的租户获取租约
+ *
+ * @param name 租约名称，无特殊要求，相同名称会竞争租约
+ */
+ void startLeaseTask(String name);
+
+ /**
+ * 申请租约，会启动一个线程，不停申请租约，直到申请成功。 申请成功后，每 租期/2 续约。 如果目前被其他租户获取租约，只有在对方租约失效，后才允许新的租户获取租约
+ *
+ * @param name 租约名称，无特殊要求，相同名称会竞争租约
+ * @param callback 当第一获取租约时，回调此函数
+ */
+ void startLeaseTask(final String name, ILeaseGetCallback callback);
+
+ /**
+ * 申请租约，会启动一个线程，不停申请租约，直到申请成功。 申请成功后，每 租期/2 续约。 如果目前被其他租户获取租约，只有在对方租约失效，后才允许新的租户获取租约
+ *
+ * @param name 租约名称，无特殊要求，相同名称会竞争租约
+ * @param leaseTermSecond 租期，在租期内可以做业务处理，单位是秒
+ * @param callback 当第一获取租约时，回调此函数
+ */
+ void startLeaseTask(final String name, int leaseTermSecond, ILeaseGetCallback callback);
+
+ /**
+ * 申请锁,无论成功与否，立刻返回。如果不释放，最大锁定时间是5分钟
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @return 是否枷锁成功
+ */
+ boolean lock(String name, String lockerName);
+
+ /**
+ * 申请锁,无论成功与否，立刻返回。默认锁定时间是5分钟
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param lockTimeSecond 如果不释放，锁定的最大时间，单位是秒
+ * @return 是否枷锁成功
+ * @return
+ */
+ boolean lock(String name, String lockerName, int lockTimeSecond);
+
+ /**
+ * 申请锁，如果没有则等待，等待时间可以指定，如果是－1 则无限等待。如果不释放，最大锁定时间是5分钟
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param waitTime 没获取锁时，最大等待多长时间，如果是－1 则无限等待
+ * @return 是否枷锁成功
+ */
+ boolean tryLocker(String name, String lockerName, long waitTime);
+
+ /**
+ * 申请锁，如果没有则等待，等待时间可以指定，如果是－1 则无限等待。如果不释放，最大锁定时间是lockTimeSecond
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param waitTime 没获取锁时，最大等待多长时间，如果是－1 则无限等待
+ * @param lockTimeSecond 如果不释放，锁定的最大时间，单位是秒
+ * @return 是否枷锁成功
+ */
+ boolean tryLocker(String name, String lockerName, long waitTime, int lockTimeSecond);
+
+ /**
+ * 释放锁
+ *
+ * @param name
+ * @param lockerName
+ * @return
+ */
+ boolean unlock(String name, String lockerName);
+
+ /**
+ * 对于已经获取锁的，可以通过这个方法，一直持有锁。 和租约的区别是，当释放锁后，无其他实例抢占。无法实现主备模式
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param lockTimeSecond 租期，这个方法会自动续约，如果不主动释放，会一直持有锁
+ * @return 是否成功获取锁
+ */
+ boolean holdLock(String name, String lockerName, int lockTimeSecond);
+
+ /**
+ * 是否持有锁，不会申请锁。如果以前申请过，且未过期，返回true，否则返回false
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @return
+ */
+ boolean hasHoldLock(String name, String lockerName);
+
+ List<LeaseInfo> queryLockedInstanceByNamePrefix(String name, String lockerNamePrefix);
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorage.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorage.java
new file mode 100644
index 0000000..9278bc0
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorage.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+
+public interface ILeaseStorage {
+
+ /**
+ * 更新lease info，需要是原子操作，存储保障多线程操作的原子性
+ *
+ * @param leaseInfo 租约表数据
+ * @return
+ */
+ boolean updateLeaseInfo(LeaseInfo leaseInfo);
+
+ /**
+ * 统计这个租约名称下，LeaseInfo对象个数
+ *
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ Integer countLeaseInfo(String leaseName);
+
+ /**
+ * 查询无效的的租约
+ *
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryInValidateLease(String leaseName);
+
+ /**
+ * 查询有效的的租约
+ *
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryValidateLease(String leaseName);
+
+ /**
+ * 按前缀查询有效的租约信息
+ *
+ * @param namePrefix
+ * @return
+ */
+ List<LeaseInfo> queryValidateLeaseByNamePrefix(String namePrefix);
+
+ /**
+ * 增加租约
+ *
+ * @param leaseInfo 租约名称，无特殊要求，相同名称会竞争租约
+ */
+ void addLeaseInfo(LeaseInfo leaseInfo);
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorasge.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorasge.java
new file mode 100644
index 0000000..cbfe26e
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorasge.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.lease.service;
+
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+
+public interface ILeaseStorasge {
+
+
+
+ /**
+ * 更新lease info，需要是原子操作，存储保障多线程操作的原子性
+ * @param leaseInfo 租约表数据
+ * @return
+ */
+ boolean updateLeaseInfo(LeaseInfo leaseInfo);
+
+ /**
+ * 统计这个租约名称下，LeaseInfo对象个数
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ Integer countLeaseInfo(String leaseName);
+
+ /**
+ * 查询无效的的租约
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryInValidateLease(String leaseName);
+
+
+ /**
+ * 查询无效的的租约
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryValidateLease(String leaseName);
+
+ /**
+ * 增加租约
+ * @param leaseInfo 租约名称，无特殊要求，相同名称会竞争租约
+ */
+ void addLeaseInfo(LeaseInfo leaseInfo);
+
+
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/BasedLesaseImpl.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/BasedLesaseImpl.java
new file mode 100644
index 0000000..21db98d
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/BasedLesaseImpl.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.impl;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.IPUtil;
+import org.apache.rocketmq.streams.common.utils.RuntimeUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+import org.apache.rocketmq.streams.lease.service.ILeaseStorage;
+
+public abstract class BasedLesaseImpl implements ILeaseService {
+ private static final Log LOG = LogFactory.getLog(BasedLesaseImpl.class);
+
+ private static final String CONSISTENT_HASH_PREFIX = "consistent_hash_";
+ private static AtomicBoolean syncStart = new AtomicBoolean(false);
+ private static final int synTime = 120; // 5分钟的一致性hash同步时间太久了，改为2分钟
+ protected ScheduledExecutorService taskExecutor = null;
+ protected int leaseTerm = 300 * 2; // 租约时间
+
+ // protected transient JDBCDriver jdbcDataSource = null;
+ protected ILeaseStorage leaseStorage;
+ protected volatile Map<String, Date> leaseName2Date = new ConcurrentHashMap<>(); // 每个lease name对应的租约到期时间
+
+ public BasedLesaseImpl() {
+
+ taskExecutor = new ScheduledThreadPoolExecutor(10);
+
+ }
+
+ /**
+ * lease_name: consistent_hash_ip, lease_user_ip: ip,定时刷新lease_info表，检查一致性hash环的节点情况
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public boolean hasLease(String name) {
+ // 内存中没有租约信息则表示 没有租约
+ Date leaseEndTime = leaseName2Date.get(name);
+ if (leaseEndTime == null) {
+ // LOG.info("内存中根据 " + name + "没有查询到租约信息，表示没有租约");
+ return false;
+ }
+ // LOG.info("查询是否有租约 name:" + name + " ,当前时间：" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())
+ // + " 租约到期时间 " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(leaseEndTime));
+ // 有租约时间，并且租约时间大于当前时间，表示有租约信息
+ if (new Date().before(leaseEndTime)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private final Map<String, AtomicBoolean> startLeaseMap = new HashMap<>();
+
+ @Override
+ public void startLeaseTask(final String name) {
+ startLeaseTask(name, this.leaseTerm, null);
+ }
+
+ @Override
+ public void startLeaseTask(final String name, ILeaseGetCallback callback) {
+ startLeaseTask(name, this.leaseTerm, callback);
+ }
+
+ @Override
+ public void startLeaseTask(final String name, int leaseTerm, ILeaseGetCallback callback) {
+ ApplyTask applyTask = new ApplyTask(leaseTerm, name, callback);
+ startLeaseTask(name, applyTask, leaseTerm / 2, true);
+ }
+
+ /**
+ * 启动定时器，定时执行任务，确保任务可重入
+ *
+ * @param name
+ * @param runnable 具体任务
+ * @param scheduleTime 调度时间
+ * @param startNow 是否立刻启动一次
+ */
+ protected void startLeaseTask(final String name, Runnable runnable, int scheduleTime, boolean startNow) {
+ AtomicBoolean isStartLease = startLeaseMap.get(name);//多次调用，只启动一次定时任务
+ if (isStartLease == null) {
+ synchronized (this) {
+ isStartLease = startLeaseMap.get(name);
+ if (isStartLease == null) {
+ isStartLease = new AtomicBoolean(false);
+ startLeaseMap.put(name, isStartLease);
+ }
+ }
+ }
+ if (isStartLease.compareAndSet(false, true)) {
+ if (startNow) {
+ runnable.run();
+ }
+ taskExecutor.scheduleWithFixedDelay(runnable, 0, scheduleTime, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * 续约任务
+ */
+ protected class ApplyTask implements Runnable {
+
+ protected String name;
+ protected int leaseTerm;
+ protected ILeaseGetCallback callback;
+
+ public ApplyTask(int leaseTerm, String name) {
+ this(leaseTerm, name, null);
+ }
+
+ public ApplyTask(int leaseTerm, String name, ILeaseGetCallback callback) {
+ this.name = name;
+ this.leaseTerm = leaseTerm;
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // LOG.info("LeaseServiceImpl name: " + name + "开始获取租约...");
+ AtomicBoolean newApplyLease = new AtomicBoolean(false);
+ Date leaseDate = applyLeaseTask(leaseTerm, name, newApplyLease);
+ if (leaseDate != null) {
+ leaseName2Date.put(name, leaseDate);
+ LOG.info("LeaseServiceImpl, name: " + name + " " + getSelfUser() + " 获取租约成功, 租约到期时间为 "
+ + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(leaseDate));
+ } else {
+ // fix.2020.08.13 这时name对应的租约可能还在有效期内,或者本机还持有租约，需要remove
+ // leaseName2Date.remove(name);
+ LOG.info("LeaseServiceImpl name: " + name + " " + getSelfUser() + " 获取租约失败 ");
+ }
+ if (newApplyLease.get() && callback != null) {
+ callback.callback(leaseDate);
+ }
+ } catch (Exception e) {
+ LOG.error(" LeaseServiceImpl name: " + name + " " + getSelfUser() + " 获取租约出现异常 ", e);
+ }
+
+ }
+ }
+
+ /**
+ * 申请租约，如果当期租约有效，直接更新一个租约周期，如果当前租约无效，先查询是否有有效的租约，如果有申请失败，否则直接申请租约
+ */
+ protected Date applyLeaseTask(int leaseTerm, String name, AtomicBoolean newApplyLease) {
+
+ // 计算下一次租约时间 = 当前时间 + 租约时长
+ Date nextLeaseDate = DateUtil.addSecond(new Date(), leaseTerm);
+
+ // 1 如果已经有租约，则更新租约时间(内存和数据库)即可
+ if (hasLease(name)) {
+ // LOG.info("用户已有租约，更新数据库和内存中的租约信息");
+ // 更新数据库
+ LeaseInfo leaseInfo = queryValidateLease(name);
+ if (leaseInfo == null) {
+ LOG.error("LeaseServiceImpl applyLeaseTask leaseInfo is null");
+ return null;
+ }
+ // fix.2020.08.13,与本机ip相等且满足一致性hash分配策略，才续约，其他情况为null
+ String leaseUserIp = leaseInfo.getLeaseUserIp();
+ if (!leaseUserIp.equals(getSelfUser())) {
+ return null;
+ }
+ leaseInfo.setLeaseEndDate(nextLeaseDate);
+ updateLeaseInfo(leaseInfo);
+ return nextLeaseDate;
+ }
+
+ // 2 没有租约情况 判断是否可以获取租约，只要租约没有被其他人获取，则说明有有效租约
+ boolean success = canGetLease(name);
+ if (!success) { // 表示被其他机器获取到了有效的租约
+ // LOG.info("其他机器获取到了有效的租约");
+ return null;
+ }
+
+ // 3 没有租约而且可以获取租约的情况，则尝试使用数据库原子更新的方式获取租约，保证只有一台机器成功获取租约，而且可以运行
+ boolean flag = tryGetLease(name, nextLeaseDate);
+ if (flag) { // 获取租约成功
+ newApplyLease.set(true);
+ return nextLeaseDate;
+ }
+ return null;
+
+ }
+
+ /**
+ * 查询数据库，自己是否在租期内或没有被其他人租用
+ *
+ * @return
+ */
+ protected boolean canGetLease(String name) {
+ LeaseInfo leaseInfo = queryValidateLease(name);
+ if (leaseInfo == null) {
+ return true;
+ }
+ // fix.2020.08.13,租约ip为本机ip，且与一致性hash分配ip一致，才是有效租约
+ String leaseUserIp = leaseInfo.getLeaseUserIp();
+ if (leaseUserIp.equals(getSelfUser())) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * 更新数据库，占用租期并更新租期时间
+ *
+ * @param time
+ */
+ protected boolean tryGetLease(String name, Date time) {
+ // LOG.info("尝试获取租约 lease name is : " + name + " 下次到期时间: "
+ // + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(time));
+ LeaseInfo validateLeaseInfo = queryValidateLease(name);
+
+ if (validateLeaseInfo == null) {// 这里有两种情况 1 数据库里面没有租约信息 2 数据库里面有租约信息但是已经过期
+ Integer count = countLeaseInfo(name);
+ if (count == null || count == 0) {// 表示现在数据库里面没有任何租约信息，插入租约成功则表示获取成功，失败表示在这一时刻其他机器获取了租约
+ // LOG.info("数据库中暂时没有租约信息，尝试原子插入租约:" + name);
+ // fix.2020.08.13,经过一致性hash计算，该名字的任务不应该在本机执行，直接返回，无需插入。只有分配到hash执行权限的机器才可以插入并获取租约
+ if (!getSelfUser().equals(getConsistentHashHost(name))) {
+ return false;
+ }
+ validateLeaseInfo = new LeaseInfo();
+ validateLeaseInfo.setLeaseName(name);
+ validateLeaseInfo.setLeaseUserIp(getSelfUser());
+ validateLeaseInfo.setLeaseEndDate(time);
+ validateLeaseInfo.setStatus(1);
+ validateLeaseInfo.setVersion(1);
+ if (insert(validateLeaseInfo)) {
+ LOG.info("数据库中暂时没有租约信息，原子插入成功，获取租约成功:" + name);
+ return true;
+ } else {
+ LOG.info("数据库中暂时没有租约信息，原子插入失败，已经被其他机器获取租约:" + name);
+ return false;
+ }
+ } else { // 表示数据库里面有一条但是无效，这里需要两台机器按照version进行原子更新，更新成功的获取租约
+ // LOG.info("数据库中有一条无效的租约信息，尝试根据版本号去原子更新租约信息:" + name);
+ LeaseInfo inValidateLeaseInfo = queryInValidateLease(name);
+ if (inValidateLeaseInfo == null) {// 说明这个时候另外一台机器获取成功了
+ LOG.info("另外一台机器获取成功了租约:" + name);
+ return false;
+ }
+ // fix.2020.08.13,机器重启之后，该名字的任务已经不分配在此机器上执行，直接返回，无需更新数据库
+ if (!getSelfUser().equals(getConsistentHashHost(name))) {
+ return false;
+ }
+ inValidateLeaseInfo.setLeaseName(name);
+ inValidateLeaseInfo.setLeaseUserIp(getSelfUser());
+ inValidateLeaseInfo.setLeaseEndDate(time);
+ inValidateLeaseInfo.setStatus(1);
+ boolean success = updateDBLeaseInfo(inValidateLeaseInfo);
+ if (success) {
+ LOG.info("LeaseServiceImpl 原子更新租约成功，当前机器获取到了租约信息:" + name);
+ } else {
+ LOG.info("LeaseServiceImpl 原子更新租约失败，租约被其他机器获取:" + name);
+ }
+ return success;
+ }
+
+ } else { // 判断是否是自己获取了租约，如果是自己获取了租约则更新时间（内存和数据库），
+ // 这里是为了解决机器重启的情况，机器重启，内存中没有租约信息，但是实际上该用户是有租约权限的
+ // fix.2020.08.13,租约的ip与本机ip相等，且满足一致性hash策略，才会被本机执行
+ String leaseUserIp = validateLeaseInfo.getLeaseUserIp();
+ if (leaseUserIp.equals(getSelfUser())) {
+ // 如果当期用户有租约信息，则更新数据库
+ validateLeaseInfo.setLeaseEndDate(time);
+ boolean hasUpdate = updateLeaseInfo(validateLeaseInfo);
+ if (hasUpdate) {
+ LOG.info(
+ "LeaseServiceImpl机器重启情况，当前用户有租约信息，并且更新数据库成功，租约信息为 name :" + validateLeaseInfo.getLeaseName()
+ + " ip : " + validateLeaseInfo.getLeaseUserIp() + " 到期时间 : " + new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss").format(validateLeaseInfo.getLeaseEndDate()));
+ return true;
+ } else {
+ LOG.info("LeaseServiceImpl 机器重启情况，当前用户有租约信息，并且更新数据库失败，表示失去租约:" + name);
+ return false;
+ }
+ }
+ // LOG.info("LeaseServiceImpl 租约被其他机器获取，租约信息为 name :" + validateLeaseInfo.getLeaseName() + " ip : "
+ // + validateLeaseInfo.getLeaseUserIp() + " 到期时间 : "
+ // + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(validateLeaseInfo.getLeaseEndDate()));
+ return false;
+ }
+
+ }
+
+ protected LeaseInfo queryValidateLease(String name) {
+ //String sql = "SELECT * FROM lease_info WHERE lease_name ='" + name + "' and status=1 and lease_end_time>now()";
+ //// LOG.info("LeaseServiceImpl query validate lease sql:" + sql);
+ //return queryLease(name, sql);
+ return leaseStorage.queryValidateLease(name);
+ }
+
+ protected List<LeaseInfo> queryValidateLeaseByNamePrefix(String namePrefix) {
+ return leaseStorage.queryValidateLeaseByNamePrefix(namePrefix);
+ }
+
+ /**
+ * 如果发生唯一索引冲突返回失败
+ *
+ * @param leaseInfo
+ * @return
+ */
+ private boolean insert(LeaseInfo leaseInfo) {
+ try {
+ addLeaseInfo(leaseInfo);
+ return true;
+ } catch (Exception e) {
+ LOG.error("LeaseServiceImpl insert error", e);
+ return false;
+ }
+ }
+
+ /**
+ * 更新时需要加version＝当前version，如果更新数据条数为0，返回false
+ *
+ * @param leaseInfo
+ * @return
+ */
+ protected boolean updateDBLeaseInfo(LeaseInfo leaseInfo) {
+ return updateLeaseInfo(leaseInfo);
+ }
+
+ protected boolean updateLeaseInfo(LeaseInfo leaseInfo) {
+
+ return leaseStorage.updateLeaseInfo(leaseInfo);
+ }
+
+ protected Integer countLeaseInfo(String name) {
+
+ return leaseStorage.countLeaseInfo(name);
+ }
+
+ protected LeaseInfo queryInValidateLease(String name) {
+
+ return leaseStorage.queryInValidateLease(name);
+ }
+
+ protected void addLeaseInfo(LeaseInfo leaseInfo) {
+
+ leaseStorage.addLeaseInfo(leaseInfo);
+
+ }
+
+ /**
+ * 本地ip地址作为自己的唯一标识
+ *
+ * @return
+ */
+ public static String getLocalName() {
+ return IPUtil.getLocalIdentification() + ":" + Optional.ofNullable(RuntimeUtil.getPid()).orElse("UNKNOWN");
+ }
+
+ /**
+ * 本地ip地址作为自己的唯一标识
+ *
+ * @return
+ */
+ public String getSelfUser() {
+ return getLocalName();
+ }
+
+ private String getConsistentHashHost(String name) {
+ //if (StringUtil.isEmpty(leaseConsistentHashSuffix)) {
+ // return getSelfUser();
+ //}
+ //return consistentHashInstance.getCandidateNode(name);
+ return getSelfUser();
+ }
+
+ public void setLeaseStorage(ILeaseStorage leaseStorage) {
+ this.leaseStorage = leaseStorage;
+ }
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/LeaseServiceImpl.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/LeaseServiceImpl.java
new file mode 100644
index 0000000..860710a
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/LeaseServiceImpl.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.impl;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class LeaseServiceImpl extends BasedLesaseImpl {
+
+ private static final Log LOG = LogFactory.getLog(LeaseServiceImpl.class);
+
+ private transient ConcurrentHashMap<String, HoldLockTask> holdLockTasks = new ConcurrentHashMap();
+
+ protected ConcurrentHashMap<String, HoldLockFunture> seizeLockingFuntures = new ConcurrentHashMap<>();
+ //如果是抢占锁状态中，则不允许申请锁
+
+ public LeaseServiceImpl() {
+ super();
+ }
+
+ /**
+ * 尝试获取锁，可以等待waitTime，如果到点未返回，则直接返回。如果是-1，则一直等待
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param waitTime 等待时间，是微秒单位
+ * @return
+ */
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime) {
+ return tryLocker(name, lockerName, waitTime, ILeaseService.DEFALUT_LOCK_TIME);
+ }
+
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime, int lockTimeSecond) {
+ long now = System.currentTimeMillis();
+ boolean success = lock(name, lockerName, lockTimeSecond);
+ while (!success) {
+ if (waitTime > -1 && (System.currentTimeMillis() - now > waitTime)) {
+ break;
+ }
+ success = lock(name, lockerName, lockTimeSecond);
+ if (success) {
+ return success;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ LOG.error("LeaseServiceImpl try locker error", e);
+ }
+ }
+ return success;
+
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName) {
+ return lock(name, lockerName, ILeaseService.DEFALUT_LOCK_TIME);
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName, int leaseSecond) {
+ lockerName = createLockName(name, lockerName);
+ Future future = seizeLockingFuntures.get(lockerName);
+ if (future != null && ((HoldLockFunture)future).isDone == false) {
+ return false;
+ }
+ Date nextLeaseDate =
+ DateUtil.addSecond(new Date(), leaseSecond);// 默认锁定5分钟，用完需要立刻释放.如果时间不同步，可能导致锁失败
+ boolean success = tryGetLease(lockerName, nextLeaseDate);// 申请锁，锁的时间是leaseTerm
+ return success;
+ }
+
+ @Override
+ public boolean unlock(String name, String lockerName) {
+ // LOG.info("LeaseServiceImpl unlock,name:" + name);
+ lockerName = createLockName(name, lockerName);
+ LeaseInfo validateLeaseInfo = queryValidateLease(lockerName);
+ if (validateLeaseInfo == null) {
+ LOG.warn("LeaseServiceImpl unlock,validateLeaseInfo is null,lockerName:" + lockerName);
+ }
+ if (validateLeaseInfo != null && validateLeaseInfo.getLeaseUserIp().equals(getSelfUser())) {
+ validateLeaseInfo.setStatus(0);
+ updateDBLeaseInfo(validateLeaseInfo);
+ }
+ HoldLockTask holdLockTask = holdLockTasks.remove(lockerName);
+ if (holdLockTask != null) {
+ holdLockTask.close();
+ }
+ leaseName2Date.remove(lockerName);
+ return false;
+ }
+
+ /**
+ * 如果有锁，则一直持有，如果不能获取，则结束。和租约不同，租约是没有也会尝试重试，一备对方挂机，自己可以接手工作
+ *
+ * @param name
+ * @param secondeName
+ * @param lockTimeSecond 获取锁的时间
+ * @return
+ */
+ @Override
+ public boolean holdLock(String name, String secondeName, int lockTimeSecond) {
+ if (hasHoldLock(name, secondeName)) {
+ return true;
+ }
+ synchronized (this) {
+ if (hasHoldLock(name, secondeName)) {
+ return true;
+ }
+ String lockerName = createLockName(name, secondeName);
+ Date nextLeaseDate =
+ DateUtil.addSecond(new Date(), lockTimeSecond);
+ boolean success = tryGetLease(lockerName, nextLeaseDate);// 申请锁，锁的时间是leaseTerm
+ if (success == false) {
+ return false;
+ }
+ leaseName2Date.put(lockerName, nextLeaseDate);
+
+ if (!holdLockTasks.containsKey(lockerName)) {
+ HoldLockTask holdLockTask = new HoldLockTask(lockTimeSecond, lockerName, this);
+ holdLockTask.start();
+ holdLockTasks.putIfAbsent(lockerName, holdLockTask);
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * 是否持有锁，不访问数据库，直接看本地
+ *
+ * @param name
+ * @param secondeName
+ * @return
+ */
+ @Override
+ public boolean hasHoldLock(String name, String secondeName) {
+ String lockerName = createLockName(name, secondeName);
+ return hasLease(lockerName);
+ }
+
+ @Override
+ public List<LeaseInfo> queryLockedInstanceByNamePrefix(String name, String lockerNamePrefix) {
+ String leaseNamePrefix = MapKeyUtil.createKey(name, lockerNamePrefix);
+ return queryValidateLeaseByNamePrefix(leaseNamePrefix);
+ }
+
+ private String createLockName(String name, String lockerName) {
+ return MapKeyUtil.createKey(name, lockerName);
+ }
+
+ private class HoldLockTask extends ApplyTask {
+ protected volatile boolean iscontinue = true;
+ protected LeaseServiceImpl leaseService;
+ protected ScheduledExecutorService scheduledExecutor;
+
+ public HoldLockTask(int leaseTerm, String name, LeaseServiceImpl leaseService) {
+ super(leaseTerm, name);
+ this.leaseService = leaseService;
+ scheduledExecutor = new ScheduledThreadPoolExecutor(1);
+
+ }
+
+ public void start() {
+ scheduledExecutor.scheduleWithFixedDelay(this, leaseTerm / 2, leaseTerm / 2, TimeUnit.SECONDS);
+ }
+
+ public void close() {
+ iscontinue = false;
+ if (scheduledExecutor != null) {
+ scheduledExecutor.shutdown();
+ }
+ }
+
+ public boolean isIscontinue() {
+ return iscontinue;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (!iscontinue) {
+ return;
+ }
+ Date leaseDate = applyLeaseTask(leaseTerm, name, new AtomicBoolean(false));
+ if (leaseDate != null) {
+ leaseName2Date.put(name, leaseDate);
+ LOG.debug("LeaseServiceImpl, name: " + name + " " + getSelfUser() + " 续约锁成功, 租约到期时间为 "
+ + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(leaseDate));
+ } else {
+ iscontinue = false;
+ synchronized (leaseService) {
+ holdLockTasks.remove(name);
+ }
+ LOG.info("LeaseServiceImpl name: " + name + " " + getSelfUser() + " 续约锁失败，续锁程序会停止");
+ }
+ } catch (Exception e) {
+ iscontinue = false;
+ LOG.error(" LeaseServiceImpl name: " + name + " " + getSelfUser() + " 续约锁出现异常，续锁程序会停止", e);
+ }
+
+ }
+
+ }
+
+ /**
+ * 抢占锁的future，必须等锁超时才能继续获取锁
+ */
+ protected class HoldLockFunture implements Future<Boolean> {
+ private volatile boolean isDone = false;
+ private volatile Date date = null;
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ throw new RuntimeException("can not cancel");
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ if (date != null && System.currentTimeMillis() - date.getTime() >= 0) {
+ isDone = true;
+ return isDone;
+ }
+ return false;
+ }
+
+ @Override
+ public Boolean get() throws InterruptedException, ExecutionException {
+ while (isDone() == false) {
+ Thread.sleep(1000);
+ }
+ return true;
+ }
+
+ private long startTime = System.currentTimeMillis();
+
+ @Override
+ public Boolean get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+
+ throw new RuntimeException("can not support timeout ");
+ }
+
+ }
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/MockLeaseImpl.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/MockLeaseImpl.java
new file mode 100644
index 0000000..d952581
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/MockLeaseImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.impl;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+
+/**
+ * 在内存和文件模式下使用，所有的申请都会返回true，主要用来做业务测试
+ */
+public class MockLeaseImpl implements ILeaseService {
+ @Override
+ public boolean hasLease(String name) {
+ return true;
+ }
+
+ @Override
+ public void startLeaseTask(String name) {
+
+ }
+
+ @Override
+ public void startLeaseTask(String name, ILeaseGetCallback callback) {
+ callback.callback(DateUtil.addMinute(new Date(), 1));
+ }
+
+ @Override
+ public void startLeaseTask(String name, int leaseTerm, ILeaseGetCallback callback) {
+
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName) {
+ return true;
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName, int lockTimeSecond) {
+ return true;
+ }
+
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime) {
+ return true;
+ }
+
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime, int lockTimeSecond) {
+ return true;
+ }
+
+ @Override
+ public boolean unlock(String name, String lockerName) {
+ return true;
+ }
+
+ @Override
+ public boolean holdLock(String name, String lockerName, int lockTimeSecond) {
+ return true;
+ }
+
+ @Override
+ public boolean hasHoldLock(String name, String lockerName) {
+ return true;
+ }
+
+ @Override
+ public List<LeaseInfo> queryLockedInstanceByNamePrefix(String name, String lockerNamePrefix) {
+ return null;
+ }
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/storages/DBLeaseStorage.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/storages/DBLeaseStorage.java
new file mode 100644
index 0000000..b99132d
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/storages/DBLeaseStorage.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.storages;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseStorage;
+
+public class DBLeaseStorage implements ILeaseStorage {
+ private static final Log LOG = LogFactory.getLog(DBLeaseStorage.class);
+ protected JDBCDriver jdbcDataSource;
+ private String url;
+ protected String userName;
+ protected String password;
+ protected String jdbc;
+
+ public DBLeaseStorage(String jdbc, String url, String userName, String password) {
+ this.jdbc = jdbc;
+ this.url = url;
+ this.userName = userName;
+ this.password = password;
+ jdbcDataSource = DriverBuilder.createDriver(jdbc, url, userName, password);
+ }
+
+ @Override
+ public boolean updateLeaseInfo(LeaseInfo leaseInfo) {
+ String sql = "UPDATE lease_info SET version=version+1,status=#{status},gmt_modified=now()";
+ String whereSQL = " WHERE id=#{id} and version=#{version}";
+
+ if (StringUtil.isNotEmpty(leaseInfo.getLeaseName())) {
+ sql += ",lease_name=#{leaseName}";
+ }
+ if (StringUtil.isNotEmpty(leaseInfo.getLeaseUserIp())) {
+ sql += ",lease_user_ip=#{leaseUserIp}";
+ }
+ if (leaseInfo.getLeaseEndDate() != null) {
+ sql += ",lease_end_time=#{leaseEndDate}";
+ }
+ sql += whereSQL;
+ sql = SQLUtil.parseIbatisSQL(leaseInfo, sql);
+ try {
+ int count = getOrCreateJDBCDataSource().update(sql);
+ boolean success = count > 0;
+ if (success) {
+ synchronized (this) {
+ leaseInfo.setVersion(leaseInfo.getVersion() + 1);
+ }
+ } else {
+ System.out.println(count);
+ }
+ return success;
+ } catch (Exception e) {
+ LOG.error("LeaseServiceImpl updateLeaseInfo excuteUpdate error", e);
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ @Override
+ public Integer countLeaseInfo(String leaseName) {
+ String sql = "SELECT count(*) as c FROM lease_info WHERE lease_name = '" + leaseName + "' and status = 1";
+ try {
+
+ List<Map<String, Object>> rows = getOrCreateJDBCDataSource().queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ Long value = (Long)rows.get(0).get("c");
+ return value.intValue();
+ } catch (Exception e) {
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ @Override
+ public LeaseInfo queryInValidateLease(String leaseName) {
+ String sql = "SELECT * FROM lease_info WHERE lease_name ='" + leaseName + "' and status=1 and lease_end_time<'" + DateUtil.getCurrentTimeString() + "'";
+ LOG.info("LeaseServiceImpl queryInValidateLease builder:" + sql);
+ return queryLease(leaseName, sql);
+ }
+
+ @Override
+ public LeaseInfo queryValidateLease(String leaseName) {
+ String sql = "SELECT * FROM lease_info WHERE lease_name ='" + leaseName + "' and status=1 and lease_end_time>now()";
+ return queryLease(leaseName, sql);
+ }
+
+ @Override
+ public List<LeaseInfo> queryValidateLeaseByNamePrefix(String namePrefix) {
+ String sql = "SELECT * FROM lease_info WHERE lease_name like '" + namePrefix + "%' and status=1 and lease_end_time>now()";
+ try {
+ List<LeaseInfo> leaseInfos = new ArrayList<>();
+ List<Map<String, Object>> rows = getOrCreateJDBCDataSource().queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ for (Map<String, Object> row : rows) {
+ LeaseInfo leaseInfo = convert(row);
+ leaseInfos.add(leaseInfo);
+ }
+
+ return leaseInfos;
+ } catch (Exception e) {
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ @Override
+ public void addLeaseInfo(LeaseInfo leaseInfo) {
+ String sql =
+ " REPLACE INTO lease_info(lease_name,lease_user_ip,lease_end_time,status,version,gmt_create,gmt_modified)"
+ + " VALUES (#{leaseName},#{leaseUserIp},#{leaseEndDate},#{status},#{version},now(),now())";
+ sql = SQLUtil.parseIbatisSQL(leaseInfo, sql);
+ try {
+
+ getOrCreateJDBCDataSource().execute(sql);
+ } catch (Exception e) {
+ LOG.error("LeaseServiceImpl execute sql error,sql:" + sql, e);
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ protected JDBCDriver getOrCreateJDBCDataSource() {
+ if (this.jdbcDataSource == null || !this.jdbcDataSource.isValidate()) {
+ synchronized (this) {
+ if (this.jdbcDataSource == null || !this.jdbcDataSource.isValidate()) {
+ this.jdbcDataSource =
+ DriverBuilder.createDriver(this.jdbc, this.url, this.userName, this.password);
+ }
+ }
+ }
+ return jdbcDataSource;
+ }
+
+ protected LeaseInfo queryLease(String name, String sql) {
+ try {
+ List<Map<String, Object>> rows = getOrCreateJDBCDataSource().queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ return convert(rows.get(0));
+ } catch (Exception e) {
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ protected LeaseInfo convert(Map<String, Object> map) {
+ LeaseInfo leaseInfo = new LeaseInfo();
+ leaseInfo.setId(getMapLongValue("id", map));
+ leaseInfo.setCreateTime(getMapDateValue("gmt_create", map));
+ leaseInfo.setLeaseEndDate(getMapDateValue("lease_end_time", map));
+ leaseInfo.setLeaseName(getMapValue("lease_name", map, String.class));
+ leaseInfo.setLeaseUserIp(getMapValue("lease_user_ip", map, String.class));
+ Integer stauts = getMapValue("status", map, Integer.class);
+ if (stauts != null) {
+ leaseInfo.setStatus(stauts);
+ }
+ leaseInfo.setUpdateTime(getMapDateValue("gmt_modified", map));
+ Long version = getMapLongValue("version", map);
+ if (version != null) {
+ leaseInfo.setVersion(version);
+ }
+ return leaseInfo;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> T getMapValue(String fieldName, Map<String, Object> map, Class<T> integerClass) {
+ Object value = map.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ return (T)value;
+ }
+
+ private Long getMapLongValue(String fieldName, Map<String, Object> map) {
+ Object value = map.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Long) {
+ return (Long)value;
+ }
+ if (value instanceof BigInteger) {
+ return ((BigInteger)value).longValue();
+ }
+ return null;
+ }
+
+ private Date getMapDateValue(String fieldName, Map<String, Object> map) {
+ Object value = map.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date)value;
+ }
+ if (value instanceof String) {
+ return DateUtil.parseTime(((String)value));
+ }
+ return null;
+
+ }
+
+}
diff --git a/rocketmq-streams-lease/src/test/java/org/apache/rocketmq/streams/lease/LeaseComponentTest.java b/rocketmq-streams-lease/src/test/java/org/apache/rocketmq/streams/lease/LeaseComponentTest.java
new file mode 100644
index 0000000..6404e9f
--- /dev/null
+++ b/rocketmq-streams-lease/src/test/java/org/apache/rocketmq/streams/lease/LeaseComponentTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease;
+
+import java.util.Date;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class LeaseComponentTest {
+
+ private String URL = "";
+ protected String USER_NAME = "";
+ protected String PASSWORD = "";
+
+ public LeaseComponentTest() {
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE, "DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
+
+ JDBCDriver driver = DriverBuilder.createDriver();
+ driver.execute(LeaseInfo.createTableSQL());
+ }
+
+ @Test
+ public void testLease() throws InterruptedException {
+ String leaseName = "lease.test";
+ int leaseTime = 5;
+ LeaseComponent.getInstance().getService().startLeaseTask(leaseName, leaseTime, new ILeaseGetCallback() {
+ @Override
+ public void callback(Date nextLeaseDate) {
+ System.out.println("I get lease");
+ }
+ });
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ }
+
+ @Test
+ public void testLock() throws InterruptedException {
+ String name = "dipper";
+ String lockName = "lease.test";
+ int leaseTime = 5;
+ boolean success = LeaseComponent.getInstance().getService().lock(name, lockName, leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(6000);
+ assertFalse(LeaseComponent.getInstance().getService().hasHoldLock(name, lockName));//超期释放
+ }
+
+ /**
+ * holdlock是一直持有锁，和租约的区别是，当释放锁后，无其他实例抢占
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testHoldLock() throws InterruptedException {
+ String name = "dipper";
+ String lockName = "lease.test";
+ int leaseTime = 6;
+ boolean success = LeaseComponent.getInstance().getService().holdLock(name, lockName, leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(8000);
+ assertTrue(LeaseComponent.getInstance().getService().hasHoldLock(name, lockName));//会自动续约，不会释放，可以手动释放
+ LeaseComponent.getInstance().getService().unlock(name, lockName);
+ assertFalse(LeaseComponent.getInstance().getService().hasHoldLock(name, lockName));
+ }
+
+ @Test
+ public void testHoldLockContinue() throws InterruptedException {
+ String name = "dipper";
+ String lockName = "lease.test";
+ int leaseTime = 6;
+ boolean success = holdLock(name, lockName, leaseTime);//锁定5秒钟
+ while (true) {
+ Thread.sleep(1000);
+ System.out.println(holdLock(name, lockName, leaseTime));
+ }
+ }
+
+ protected boolean holdLock(String name, String lockName, int leaseTime) {
+ ILeaseService leaseService = LeaseComponent.getInstance().getService();
+ if (leaseService.hasHoldLock(name, lockName)) {
+ return true;
+ }
+
+ boolean success = leaseService.holdLock(name, lockName, leaseTime);
+ return success;
+ }
+
+}
diff --git a/rocketmq-streams-lease/src/test/resources/log4j.xml b/rocketmq-streams-lease/src/test/resources/log4j.xml
new file mode 100755
index 0000000..7812fe7
--- /dev/null
+++ b/rocketmq-streams-lease/src/test/resources/log4j.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
[rocketmq-streams] 03/15: add lease、dim and client module
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 8c20f7314b838d853de39c90d19fd2e509a44713
Merge: b757f8d 997f3df
Author: 刈刀 <ju...@alibaba-inc.com>
AuthorDate: Mon Aug 2 11:30:22 2021 +0800
add lease、dim and client module
README.md | 28 +++++++++++++---------------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --cc README.md
index 5f3cde5,49cd84d..2dfb623
--- a/README.md
+++ b/README.md
@@@ -1,108 -1,1 +1,106 @@@
-# rocketmq-streams
+# Rocketmq Streams
+
+## Features
+
+* 轻量级部署：可以单独部署，也支持集群部署
+* 多种类型的数据输入以及输出，source支持 rocketmq ， sink支持db, rocketmq 等
+
+## DataStream Example
+
+```java
+import org.apache.rocketmq.streams.client.transform.DataStream;
+
+DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+
+ source
+ .fromFile("/Users/junjie.cheng/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
+```
+
+## Maven Repository
+
+```xml
+
+<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+</dependency>
+```
+
+# Core API
+
+rocketmq-stream 实现了一系列高级的API，可以让用户很方便的编写流计算的程序，实现自己的业务需求；
+
+## StreamBuilder
+
+StreamBuilder 用于构建流任务的源； 内部包含```dataStream()```和```tableStream()```俩个方法，分别返回DataStreamSource和TableStreamSource俩个源；
+
++ [dataStream(nameSpaceName,pipelineName)]() 返回DataStreamSource实例，用于分段编程实现流计算任务；
+++ [tableStream(nameSpaceName,pipelineName)]()返回TableStreamSource实例， 用于脚本编程实现流计算任务；
+
+## DataStream API
+
+### Source
-
+DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据；
+
++ ```fromFile``` 从文件中读取数据， 该方法包含俩个参数
+ + ```filePath``` 文件路径，必填参数
+ + ```isJsonData``` 是否json数据， 非必填参数， 默认为```true```
+
+
++ ```fromRocketmq``` 从rocketmq中获取数据，包含四个参数
+ + ```topic``` rocketmq消息队列的topic名称，必填参数
+ + ```groupName``` 消费者组的名称，必填参数
+ + ```isJson``` 是否json格式，非必填参数
+ + ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
+
+
++ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
+
+### transform
-
+transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
+
+#### DataStream
-
+DataStream实现了一系列常见的流计算算子
+
++ ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream
++ ```flatmap``` 与map类似，一个输入项对应0个或者多个输出项
++ ```filter``` 只选择func返回true的源DStream的记录来返回一个新的DStream
++ ```forEach``` 对每个记录执行一次函数func， 返回一个新的DataStream
++ ```selectFields``` 对每个记录返回对应的字段值，返回一个新的DataStream
++ ```operate``` 对每个记录执行一次自定义的函数，返回一个新的DataStream
++ ```script``` 针对每个记录的字段执行一段脚本，返回新的字段，生成一个新的DataStream
++ ```toPrint``` 将结果在控制台打印，生成新的DataStreamAction实例
++ ```toFile``` 将结果保存为文件，生成一个新的DataStreamAction实例
++ ```toDB``` 将结果保存到数据库
++ ```toRocketmq``` 将结果输出到rocketmq
++ ```toSls``` 将结果输出到sls
++ ```to``` 将结果经过自定义的ISink接口输出到指定的存储
++ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
- + ```count``` 在窗口内计数
- + ```min``` 获取窗口内统计值的最小值
- + ```max``` 获取窗口内统计值得最大值
- + ```avg``` 获取窗口内统计值的平均值
- + ```sum``` 获取窗口内统计值的加和值
- + ```reduce``` 在窗口内进行自定义的汇总运算
++ + ```count``` 在窗口内计数
++ + ```min``` 获取窗口内统计值的最小值
++ + ```max``` 获取窗口内统计值得最大值
++ + ```avg``` 获取窗口内统计值的平均值
++ + ```sum``` 获取窗口内统计值的加和值
++ + ```reduce``` 在窗口内进行自定义的汇总运算
++ ```join``` 根据条件将将俩个流进行关联， 合并为一个大流进行相关的运算
++ ```union``` 将俩个流进行合并
++ ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算
++ ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等
+
- # Strategy
+
++# Strategy
+策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
+
+```java
+//指定checkpoint的存储策略
+source
- .fromRocketmq("TSG_META_INFO","")
- .map(message->message+"--")
- .toPrint(1)
- .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
- .start();
++.fromRocketmq("TSG_META_INFO", "")
++.map(message -> message + "--")
++.toPrint(1)
++.with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX", "", "", 0L))
++.start();
+```
[rocketmq-streams] 14/15: Merge pull request #6 from RonzL/main
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit f349dd946ba29bdbc05ae958602c8b6936476d50
Merge: 6d89c9a 9c898ec
Author: Heng Du <du...@apache.org>
AuthorDate: Mon Aug 2 14:59:15 2021 +0800
Merge pull request #6 from RonzL/main
add module db-operator、transport-minio
rocketmq-streams-db-operator/pom.xml | 34 ++
.../rocketmq-streams-db-operator.iml | 16 +
.../streams/db/configuable/DBConfigureService.java | 282 ++++++++++++
.../DBSupportParentConfigureService.java | 37 ++
.../rocketmq/streams/db/driver/DriverBuilder.java | 111 +++++
.../rocketmq/streams/db/driver/IDriverBudiler.java | 36 ++
.../rocketmq/streams/db/driver/JDBCDriver.java | 277 ++++++++++++
.../db/driver/batchloader/BatchRowLoader.java | 179 ++++++++
.../db/driver/batchloader/IRowOperator.java | 33 ++
.../rocketmq/streams/db/driver/orm/ORMUtil.java | 490 +++++++++++++++++++++
.../rocketmq/streams/db/operator/SQLOperator.java | 178 ++++++++
.../org/apache/rocketmq/streams/db/Person.java | 110 +++++
.../DBSupportParentConfigureServiceTest.java | 74 ++++
.../streams/db/driver/orm/ORMUtilTest.java | 86 ++++
rocketmq-streams-transport-minio/pom.xml | 25 ++
.../rocketmq-streams-transport-minio.iml | 17 +
.../transport/minio/MinioFileTransport.java | 141 ++++++
.../yundun/dipper/configurable/DataTpyeTest.java | 70 +++
.../streams/configuable/model/DataTpyeTest.java | 68 +++
.../rocketmq/streams/configuable/model/Person.java | 97 ++++
.../streams/configurable/model/Person.java | 97 ++++
.../component/ConfigurableComponent.properties | 7 +
.../src/test/resources/log4j.xml | 20 +
.../src/test/resources/pro-function.txt | 11 +
.../src/test/resources/python_script.py | 22 +
25 files changed, 2518 insertions(+)
[rocketmq-streams] 08/15: add module db-operator、transport-minio
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 9c898ecf0c5301f1ae1d0c144f15f96d2151ba96
Author: muyang <li...@alibaba-inc.com>
AuthorDate: Mon Aug 2 12:21:28 2021 +0800
add module db-operator、transport-minio
---
rocketmq-streams-db-operator/pom.xml | 34 ++
.../rocketmq-streams-db-operator.iml | 16 +
.../streams/db/configuable/DBConfigureService.java | 282 ++++++++++++
.../DBSupportParentConfigureService.java | 37 ++
.../rocketmq/streams/db/driver/DriverBuilder.java | 111 +++++
.../rocketmq/streams/db/driver/IDriverBudiler.java | 36 ++
.../rocketmq/streams/db/driver/JDBCDriver.java | 277 ++++++++++++
.../db/driver/batchloader/BatchRowLoader.java | 179 ++++++++
.../db/driver/batchloader/IRowOperator.java | 33 ++
.../rocketmq/streams/db/driver/orm/ORMUtil.java | 490 +++++++++++++++++++++
.../rocketmq/streams/db/operator/SQLOperator.java | 178 ++++++++
.../org/apache/rocketmq/streams/db/Person.java | 110 +++++
.../DBSupportParentConfigureServiceTest.java | 74 ++++
.../streams/db/driver/orm/ORMUtilTest.java | 86 ++++
rocketmq-streams-transport-minio/pom.xml | 25 ++
.../rocketmq-streams-transport-minio.iml | 17 +
.../transport/minio/MinioFileTransport.java | 141 ++++++
.../yundun/dipper/configurable/DataTpyeTest.java | 70 +++
.../streams/configuable/model/DataTpyeTest.java | 68 +++
.../rocketmq/streams/configuable/model/Person.java | 97 ++++
.../streams/configurable/model/Person.java | 97 ++++
.../component/ConfigurableComponent.properties | 7 +
.../src/test/resources/log4j.xml | 20 +
.../src/test/resources/pro-function.txt | 11 +
.../src/test/resources/python_script.py | 22 +
25 files changed, 2518 insertions(+)
diff --git a/rocketmq-streams-db-operator/pom.xml b/rocketmq-streams-db-operator/pom.xml
new file mode 100755
index 0000000..9a9b17b
--- /dev/null
+++ b/rocketmq-streams-db-operator/pom.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-db-operator</artifactId>
+ <name>ROCKETMQ STREAMS :: db-operator</name>
+ <packaging>jar</packaging>
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-configurable</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </dependency>
+
+
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-db-operator/rocketmq-streams-db-operator.iml b/rocketmq-streams-db-operator/rocketmq-streams-db-operator.iml
new file mode 100644
index 0000000..38ffb14
--- /dev/null
+++ b/rocketmq-streams-db-operator/rocketmq-streams-db-operator.iml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+ <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_5">
+ <output url="file://$MODULE_DIR$/${project.build.directory}/classes" />
+ <output-test url="file://$MODULE_DIR$/${project.build.directory}/test-classes" />
+ <content url="file://$MODULE_DIR$">
+ <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+ <excludeFolder url="file://$MODULE_DIR$/${project.build.directory}/classes" />
+ <excludeFolder url="file://$MODULE_DIR$/${project.build.directory}/test-classes" />
+ <excludeFolder url="file://$MODULE_DIR$/target" />
+ </content>
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ </component>
+</module>
\ No newline at end of file
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBConfigureService.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBConfigureService.java
new file mode 100644
index 0000000..ef319eb
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBConfigureService.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.configuable;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.configurable.AbstractConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IConfigurable;
+import org.apache.rocketmq.streams.configurable.service.AbstractConfigurableService;
+import org.apache.rocketmq.streams.configurable.model.Configure;
+
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.interfaces.IPropertyEnable;
+import org.apache.rocketmq.streams.common.utils.AESUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+
+import java.util.*;
+
+/**
+ * Configuable对象存储在db中，是生成环境常用的一种模式 数据库参数可以配置在配置文件中，ConfiguableComponent在启动时，会把参数封装在Properties中，调用DBConfigureService(Properties properties) 构造方法完成实例创建
+ */
+
+public class DBConfigureService extends AbstractConfigurableService implements IPropertyEnable {
+
+ private static final Log LOG = LogFactory.getLog(DBConfigureService.class);
+ private String jdbcdriver;
+ private String url;
+ private String userName;
+ private String password;
+ private String tableName = "dipper_configure";
+ @Deprecated
+ private boolean isCompatibilityOldRuleEngine = false;//兼容老规则引擎使用，正常场景不需要理会
+
+ public DBConfigureService(String jdbcdriver, String url, String userName, String password) {
+ this(jdbcdriver, url, userName, password, null);
+ }
+
+ public DBConfigureService(String jdbcdriver, String url, String userName, String password, String tableName) {
+ this.url = url;
+ this.jdbcdriver = jdbcdriver;
+ this.userName = userName;
+ this.password = password;
+ this.tableName = tableName;
+ LOG.info("DBConfigureService resource ,the info is: driver:" + this.jdbcdriver + ",url:" + this.url
+ + ",username:" + userName + ",password:" + password);
+ regJdbcDriver(jdbcdriver);
+ }
+
+ public DBConfigureService() {
+ }
+
+ /**
+ * @param properties
+ */
+ public DBConfigureService(Properties properties) {
+ super(properties);
+ initProperty(properties);
+ }
+
+ @Override
+ protected GetConfigureResult loadConfigurable(String namespace) {
+ GetConfigureResult result = new GetConfigureResult();
+ try {
+ List<Configure> configures = selectOpening(namespace);
+ List<IConfigurable> configurables = convert(configures);
+ result.setConfigurables(configurables);
+ result.setQuerySuccess(true);// 该字段标示查询是否成功，若不成功则不会更新配置
+ } catch (Exception e) {
+ result.setQuerySuccess(false);
+ LOG.error("load configurable error ", e);
+ }
+ return result;
+ }
+
+ protected List<Configure> selectOpening(String namespace) {
+ return queryConfigureByNamespace(namespace);
+ }
+
+ protected List<Configure> queryConfigureByNamespace(String... namespaces) {
+ return queryConfigureByNamespaceInner(null, namespaces);
+ }
+
+ protected List<Configure> queryConfigureByNamespaceInner(String type, String... namespaces) {
+ JDBCDriver resource = createResouce();
+ try {
+ String namespace = "namespace";
+ if (isCompatibilityOldRuleEngine && AbstractComponent.JDBC_COMPATIBILITY_RULEENGINE_TABLE_NAME.equals(tableName)) {
+ namespace = "name_space";
+ }
+ String sql = "SELECT * FROM `" + tableName + "` WHERE " + namespace + " in (" + SQLUtil.createInSql(namespaces) + ") and status =1";
+ if (StringUtil.isNotEmpty(type)) {
+ sql = sql + " and type='" + type + "'";
+ }
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("namespace", MapKeyUtil.createKeyBySign(",", namespaces));
+ sql = SQLUtil.parseIbatisSQL(jsonObject, sql);
+ // String builder = "SELECT * FROM `" + tableName + "` WHERE namespace ='" + namespace + "' and status =1";
+ List<Map<String, Object>> result = resource.queryForList(sql);
+ if (result == null) {
+ return new ArrayList<Configure>();
+ }
+ // LOG.info("load configurable's count is " + result.size());
+ return convert2Configure(result);
+ } finally {
+ if (resource != null) {
+ resource.destroy();
+ }
+ }
+ }
+
+ @Override
+ public List<IConfigurable> queryConfiguableByNamespace(String... namespaces) {
+ List<Configure> configures = queryConfigureByNamespace(namespaces);
+ List<IConfigurable> configurables = convert(configures);
+ return configurables;
+ }
+
+ public static void main(String[] args) {
+ String[] namespaces = new String[] {"rule1", null};
+ String sql = "SELECT * FROM `dipper_configure` WHERE namespace in (" + SQLUtil.createInSql(namespaces) + ") and status =1";
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("namespace", MapKeyUtil.createKeyBySign(",", namespaces));
+ sql = SQLUtil.parseIbatisSQL(jsonObject, sql);
+ System.out.println(sql);
+ }
+
+ protected void saveOrUpdate(IConfigurable configure) {
+ JDBCDriver jdbcDataSource = createResouce();
+ String sql = AbstractConfigurable.createSQL(configure, this.tableName);
+ try {
+ jdbcDataSource.executeInsert(sql);
+ } catch (Exception e) {
+ LOG.error("DBConfigureService saveOrUpdate error,sqlnode:" + sql);
+ throw new RuntimeException(e);
+ } finally {
+ if (jdbcDataSource != null) {
+ jdbcDataSource.destroy();
+ }
+ }
+ }
+
+ protected List<Configure> convert2Configure(List<Map<String, Object>> rows) {
+ List<Configure> configures = new ArrayList<Configure>();
+ for (Map<String, Object> row : rows) {
+ Configure configure = new Configure();
+ Long id = getColumnValue(row, "id");
+ configure.setId(id);
+ Date create = getColumnValue(row, "gmt_create");
+ configure.setGmtCreate(create);
+ Date modify = getColumnValue(row, "gmt_modified");
+ configure.setGmtModified(modify);
+ String namespace = getColumnValue(row, "namespace");
+ if (StringUtil.isEmpty(namespace)) {
+ namespace = getColumnValue(row, "name_space");
+ }
+ configure.setNameSpace(namespace);
+ String type = getColumnValue(row, "type");
+ configure.setType(type);
+ String name = getColumnValue(row, "name");
+ configure.setName(name);
+ String jsonValue = getColumnValue(row, "json_value");
+ try {
+ jsonValue = AESUtil.aesDecrypt(jsonValue, ConfigureFileKey.SECRECY);
+ } catch (Exception e) {
+ LOG.error("can't decrypt the value, reason:\t" + e.getCause());
+ throw new RuntimeException(e);
+ }
+ configure.setJsonValue(jsonValue);
+ configures.add(configure);
+ }
+ return configures;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected <T> T getColumnValue(Map<String, Object> row, String columnName) {
+ Object value = row.get(columnName);
+ if (value == null) {
+ return null;
+ }
+ if (java.math.BigInteger.class.isInstance(value)) {
+ return (T)Long.valueOf(value.toString());
+ }
+ return (T)value;
+
+ }
+
+ protected JDBCDriver createResouce() {
+ JDBCDriver resource = DriverBuilder.createDriver(this.jdbcdriver, this.url, this.userName, this.password);
+ return resource;
+ }
+
+ public void setJdbcdriver(String jdbcdriver) {
+ this.jdbcdriver = jdbcdriver;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ private void regJdbcDriver(String jdbcdriver) {
+ try {
+ if (StringUtil.isEmpty(jdbcdriver)) {
+ jdbcdriver = AbstractComponent.DEFAULT_JDBC_DRIVER;
+ }
+ Class.forName(jdbcdriver);
+ } catch (ClassNotFoundException e) {
+ LOG.error("DBConfigureService regJdbcDriver ClassNotFoundException error", e);
+ } catch (Exception e) {
+ LOG.error("DBConfigureService regJdbcDriver error", e);
+ }
+ }
+
+ @Override
+ public void initProperty(Properties properties) {
+ this.jdbcdriver = properties.getProperty(AbstractComponent.JDBC_DRIVER);
+ regJdbcDriver(jdbcdriver);
+ this.url = properties.getProperty(AbstractComponent.JDBC_URL);
+ this.userName = properties.getProperty(AbstractComponent.JDBC_USERNAME);
+ this.password = properties.getProperty(AbstractComponent.JDBC_PASSWORD);
+ String tableName = properties.getProperty(AbstractComponent.JDBC_TABLE_NAME);
+ String isCompatibilityOldRuleEngine = properties.getProperty(AbstractComponent.JDBC_COMPATIBILITY_OLD_RULEENGINE);
+ if (StringUtil.isNotEmpty(isCompatibilityOldRuleEngine)) {
+ this.isCompatibilityOldRuleEngine = true;
+ }
+ if (StringUtil.isNotEmpty(tableName)) {
+ this.tableName = tableName;
+ }
+ LOG.info(
+ "Properties resource ,the info is: driver:" + this.jdbcdriver + ",url:" + this.url + ",username:" + userName
+ + ",password:" + password);
+ }
+
+ @Override
+ protected void insertConfigurable(IConfigurable configurable) {
+ saveOrUpdate(configurable);
+ }
+
+ @Override
+ protected void updateConfigurable(IConfigurable configurable) {
+ saveOrUpdate(configurable);
+ }
+
+ @Override
+ public <T extends IConfigurable> List<T> loadConfigurableFromStorage(String type) {
+
+ List<Configure> configures = queryConfigureByNamespaceInner(type, namespace);
+ List<IConfigurable> configurables = convert(configures);
+ List<T> result = new ArrayList<>();
+ for (IConfigurable configurable : configurables) {
+ result.add((T)configurable);
+ }
+ return result;
+ }
+}
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java
new file mode 100644
index 0000000..77b82e0
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.configuable;
+
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import com.google.auto.service.AutoService;
+import org.apache.rocketmq.streams.configurable.service.AbstractSupportParentConfigureService;
+
+import java.util.Properties;
+
+@AutoService(IConfigurableService.class)
+@ServiceName(ConfigurableServcieType.DEFAULT_SERVICE_NAME)
+public class DBSupportParentConfigureService extends AbstractSupportParentConfigureService {
+
+ @Override
+ protected void initBeforeInitConfigurable(Properties property) {
+ this.parentConfigureService = new DBConfigureService(property);
+ this.configureService = new DBConfigureService(property);
+
+ }
+}
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/DriverBuilder.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/DriverBuilder.java
new file mode 100644
index 0000000..c0e9f53
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/DriverBuilder.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * 创建JDBCDriver，如果没有
+ */
+public class DriverBuilder {
+
+ private static final Log LOG = LogFactory.getLog(DriverBuilder.class);
+
+ public static final String DEFALUT_JDBC_DRIVER = "com.mysql.jdbc.Driver";
+
+ private static final Map<String, JDBCDriver> dataSourceMap = new ConcurrentHashMap<>();
+
+ private static AtomicInteger count = new AtomicInteger(0);
+
+ /**
+ * 使用ConfiguableComponent在属性文件配置的jdbc信息，dipper默认都是使用这个数据库连接 如果需要连接其他库，需要使用带参数的createDriver
+ *
+ * @return
+ */
+ public static JDBCDriver createDriver() {
+ String driver = ComponentCreator.getProperties().getProperty(AbstractComponent.JDBC_DRIVER);
+ String url = ComponentCreator.getProperties().getProperty(AbstractComponent.JDBC_URL);
+ String userName = ComponentCreator.getProperties().getProperty(AbstractComponent.JDBC_USERNAME);
+ String password = ComponentCreator.getProperties().getProperty(AbstractComponent.JDBC_PASSWORD);
+ return createDriver(driver, url, userName, password);
+ }
+
+ /**
+ * 根据数据库连接信息创建连接，并返回JDBCDriver
+ *
+ * @param driver 数据库驱动，如果为null，默认为mysql
+ * @param url 数据库连接url
+ * @param userName 用户名
+ * @param password 密码
+ * @return JDBCDriver
+ */
+ public static JDBCDriver createDriver(String driver, final String url, final String userName,
+ final String password) {
+ if (StringUtil.isEmpty(driver)) {
+ driver = DEFALUT_JDBC_DRIVER;
+ }
+ String className = ComponentCreator.getDBProxyClassName();
+ if (StringUtil.isNotEmpty(className)) {
+ Class clazz = ReflectUtil.forClass(className);
+ try {
+ Constructor constructor = clazz.getConstructor(
+ new Class[] {String.class, String.class, String.class, String.class});
+ JDBCDriver abstractDBDataSource = (JDBCDriver)constructor.newInstance(url, userName, password,
+ driver);
+ abstractDBDataSource.init();
+ return abstractDBDataSource;
+ } catch (Exception e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+
+ final String jdbcdriver = driver;
+ ReflectUtil.forClass(jdbcdriver);
+ JDBCDriver resource = new JDBCDriver();
+ LOG.debug("jdbcdriver=" + jdbcdriver + ",url=" + url);
+ resource.setJdbcDriver(jdbcdriver);
+ resource.setUrl(url);
+ resource.setUserName(userName);
+ resource.setPassword(password);
+ resource.init();
+ return resource;
+ }
+
+ /**
+ * 生成拼接字符串
+ *
+ * @param url
+ * @param userName
+ * @param password
+ * @return
+ */
+ private static String genereateKey(String url, String userName, String password) {
+ return url + "_" + userName + "_" + password;
+ }
+
+}
+
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/IDriverBudiler.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/IDriverBudiler.java
new file mode 100644
index 0000000..6be77eb
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/IDriverBudiler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver;
+
+import org.apache.rocketmq.streams.common.dboperator.IDBDriver;
+
+/**
+ * 返回操作数据库的driver对象，并且提供方法，判断driver是否有效，以及销毁的方法
+ */
+public interface IDriverBudiler {
+
+ /**
+ * 和dipper系统同数据源
+ *
+ * @return
+ */
+ IDBDriver createDBDriver();
+
+ boolean isValidate();
+
+ void destroy();
+}
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/JDBCDriver.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/JDBCDriver.java
new file mode 100644
index 0000000..356bce6
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/JDBCDriver.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver;
+
+import org.apache.rocketmq.streams.common.channel.sink.ISink;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.dboperator.IDBDriver;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.datasource.SingleConnectionDataSource;
+import org.springframework.jdbc.support.GeneratedKeyHolder;
+import org.springframework.jdbc.support.KeyHolder;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 数据库常用操作的封装，核心实现的接口是IJdbcTemplate 这个对象实现了IConfigurable接口，可以序列化存储和网络传输 数据库参数，可以配置成名字，实际值在配置文件配置
+ * <p>
+ */
+public class JDBCDriver extends BasedConfigurable implements IDriverBudiler, IDBDriver {
+ private String jdbcDriver = DriverBuilder.DEFALUT_JDBC_DRIVER;
+ @ENVDependence
+ protected String url;
+ @ENVDependence
+ protected String userName;
+ @ENVDependence
+ protected String password;
+
+ protected transient javax.sql.DataSource dataSource;
+ private transient IDBDriver dbDriver = null;
+
+ public JDBCDriver(String url, String userName, String password,
+ String driver) {
+ setType(ISink.TYPE);
+ this.url = url;
+ this.userName = userName;
+ this.password = password;
+ if (StringUtil.isNotEmpty(driver)) {
+ this.jdbcDriver = driver;
+ }
+ }
+
+ public JDBCDriver() {
+ setType(ISink.TYPE);
+ }
+
+ protected IDBDriver createOrGetDriver() {
+ if (dbDriver == null) {
+ synchronized (this) {
+ if (dbDriver == null) {
+ dbDriver = createDBDriver();
+ if (dataSource == null) {
+ dataSource = createDBDataSource();
+ }
+ }
+ }
+ }
+ return dbDriver;
+ }
+
+ @Override
+ public IDBDriver createDBDriver() {
+ javax.sql.DataSource dataSource = createDBDataSource();
+ return new IDBDriver() {
+ private final JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
+
+ @Override
+ public int update(String sql) {
+ return jdbcTemplate.update(sql);
+ }
+
+ @Override
+ public void execute(String sql) {
+ jdbcTemplate.execute(sql);
+ }
+
+ @Override
+ public List<Map<String, Object>> queryForList(String sql) {
+ return jdbcTemplate.queryForList(sql);
+ }
+
+ @Override
+ public Map<String, Object> queryOneRow(String sql) {
+ return jdbcTemplate.queryForMap(sql);
+ }
+
+ @Override
+ public long executeInsert(String sql) {
+ try {
+ KeyHolder keyHolder = new GeneratedKeyHolder();
+ jdbcTemplate.update(con -> con.prepareStatement(sql, PreparedStatement.RETURN_GENERATED_KEYS), keyHolder);
+ if (keyHolder.getKeyList() == null || keyHolder.getKeyList().size() > 1 || keyHolder.getKey() == null) {
+ return 0;
+ }
+ return keyHolder.getKey().longValue();
+ } catch (Exception e) {
+ String errorMsg = "execute builder error ,the builder is " + sql + ". the error msg is " + e.getMessage();
+ throw new RuntimeException(errorMsg, e);
+ }
+ }
+
+ @Override
+ public void executSqls(String... sqls) {
+ jdbcTemplate.batchUpdate(sqls);
+ }
+
+ @Override
+ public void executSqls(Collection<String> sqlCollection) {
+ if (sqlCollection == null || sqlCollection.size() == 0) {
+ return;
+ }
+ String[] sqls = new String[sqlCollection.size()];
+ int i = 0;
+ Iterator<String> it = sqlCollection.iterator();
+ while (it.hasNext()) {
+ String sql = it.next();
+ sqls[i] = sql;
+ i++;
+ }
+ executSqls(sqls);
+ }
+
+ /**
+ * 分批获取数据，最终获取全量数据
+ * @param sql 可执行的SQL
+ * @return 结果数据
+ */
+ @Override
+ public List<Map<String, Object>> batchQueryBySql(String sql, int batchSize) {
+ List<Map<String, Object>> rows = new ArrayList<>();
+ int startBatch;
+ String baseSql = sql;
+ if (sql.contains(";")) {
+ baseSql = sql.substring(0, sql.indexOf(";"));
+ }
+ String batchSQL = baseSql + " limit 0," + batchSize;
+ List<Map<String, Object>> batchResult = queryForList(batchSQL);
+ int index = 1;
+ while (batchResult.size() >= batchSize) {
+ rows.addAll(batchResult);
+ startBatch = batchSize * index;
+ batchSQL = baseSql + " limit " + startBatch + "," + batchSize;
+ batchResult = queryForList(batchSQL);
+ index++;
+ }
+ rows.addAll(batchResult);
+
+ return rows;
+ }
+ };
+ }
+
+ protected javax.sql.DataSource createDBDataSource() {
+
+ SingleConnectionDataSource dataSource = new SingleConnectionDataSource(url, userName, password, true);
+
+ dataSource.setDriverClassName(jdbcDriver);// add by 林行0221
+ // 专有云落地运维中心联调时发现独立打包时必须加这句话，否则会报找不到驱动。(MYSQL驱动包虽然已经打进去了但还要在这里显示指定)
+ dataSource.setSuppressClose(true);
+ this.dataSource = dataSource;
+ return dataSource;
+ }
+
+ @Override
+ public boolean isValidate() {
+ try {
+ if (dataSource == null) {
+ dataSource = createDBDataSource();
+ }
+ dataSource.getConnection();
+ } catch (SQLException e) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void destroy() {
+ if (dataSource instanceof SingleConnectionDataSource) {
+ SingleConnectionDataSource data = (SingleConnectionDataSource)dataSource;
+ data.destroy();
+ }
+ }
+
+ public String getJdbcDriver() {
+ return jdbcDriver;
+ }
+
+ public void setJdbcDriver(String jdbcDriver) {
+ this.jdbcDriver = jdbcDriver;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ @Override
+ public int update(String sql) {
+ return createOrGetDriver().update(sql);
+ }
+
+ @Override
+ public void execute(String sql) {
+ createOrGetDriver().execute(sql);
+ }
+
+ @Override
+ public List<Map<String, Object>> queryForList(String sql) {
+ return createOrGetDriver().queryForList(sql);
+ }
+
+ @Override
+ public Map<String, Object> queryOneRow(String sql) {
+ return createOrGetDriver().queryOneRow(sql);
+ }
+
+ @Override
+ public long executeInsert(String sql) {
+ return createOrGetDriver().executeInsert(sql);
+ }
+
+ @Override
+ public void executSqls(String... sqls) {
+ createOrGetDriver().executSqls(sqls);
+ }
+
+ @Override
+ public void executSqls(Collection<String> sqls) {
+ createOrGetDriver().executSqls(sqls);
+ }
+
+ @Override
+ public List<Map<String, Object>> batchQueryBySql(String sql, int batchSize) {
+ return createOrGetDriver().batchQueryBySql(sql, batchSize);
+ }
+}
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/batchloader/BatchRowLoader.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/batchloader/BatchRowLoader.java
new file mode 100644
index 0000000..3e4548c
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/batchloader/BatchRowLoader.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver.batchloader;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+
+/**
+ * 多线程批量加载数据，每加载一批数据后，通过IRowOperator回调接口处理数据 需要有递增的字段，这个字段有索引，不重复，如id字段
+ */
+public class BatchRowLoader {
+ private static final Log LOG = LogFactory.getLog(BatchRowLoader.class);
+ protected static final int MAX_LINE = 5000;//每个批次最大行数，根据这个值划分并行任务
+ protected static ExecutorService executorService = null;
+ protected String idFieldName;//配置字段名称，这个字段的值是数字的，且是递增的
+ protected String sql;//查询的sql语句，类似select * from table where idFieldName>#{idFieldName=0} order by idFieldName.不要加limit，系统会自动添加
+ protected int batchSize = 1000;//每批从数据库加载的数据量
+ protected IRowOperator dataRowProcessor;//加载的数据由这个回调接口处理
+ private JDBCDriver jdbcDriver;
+
+ public BatchRowLoader(String idFieldName, String sql, IRowOperator dataRowProcessor) {
+ this.idFieldName = idFieldName;
+ this.sql = sql;
+ this.dataRowProcessor = dataRowProcessor;
+ this.jdbcDriver = DriverBuilder.createDriver();
+ executorService = new ThreadPoolExecutor(20, 20,
+ 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(1000));
+ }
+
+ public void startLoadData() {
+ try {
+ String statisticalSQL = sql;
+ int startIndex = sql.toLowerCase().indexOf("from");
+ statisticalSQL = "select count(1) as c, min(" + idFieldName + ") as min, max(" + idFieldName + ") as max "
+ + sql.substring(startIndex);
+ List<Map<String, Object>> rows = jdbcDriver.queryForList(statisticalSQL);
+ Map<String, Object> row = rows.get(0);
+ int count = Integer.valueOf(row.get("c").toString());
+ if (count == 0) {
+ LOG.warn("there is no data during execute sql: " + statisticalSQL);
+ return;
+ }
+
+ IntValueKV intValueKV = new IntValueKV(count);
+ //int maxBatch=count/maxSyncCount;//每1w条数据，一个并发。如果数据量比较大，为了提高性能，并行执行
+
+ long min = Long.valueOf(row.get("min").toString());
+ long max = Long.valueOf(row.get("max").toString());
+ int maxSyncCount = count / MAX_LINE + 1;
+ long step = (max - min + 1) / maxSyncCount;
+ CountDownLatch countDownLatch = new CountDownLatch(maxSyncCount + 1);
+ AtomicInteger finishedCount = new AtomicInteger(0);
+ String taskSQL = null;
+ if (sql.indexOf(" where ") != -1) {
+ taskSQL = sql + " and " + idFieldName + ">#{startIndex} and " + idFieldName + "<=#{endIndex} order by "
+ + idFieldName + " limit " + batchSize;
+ } else {
+ taskSQL = sql + " where " + idFieldName + ">#{startIndex} and " + idFieldName
+ + "<=#{endIndex} order by " + idFieldName + " limit " + batchSize;
+ }
+
+ int i = 0;
+ for (; i < maxSyncCount; i++) {
+ FetchDataTask
+ fetchDataTask = new FetchDataTask(taskSQL, (min - 1) + step * i,
+ (min - 1) + step * (i + 1), countDownLatch, finishedCount, jdbcDriver, count);
+ executorService.execute(fetchDataTask);
+ }
+ FetchDataTask
+ fetchDataTask = new FetchDataTask(taskSQL, (min - 1) + step * i, (min - 1) + step * (i + 1),
+ countDownLatch, finishedCount, jdbcDriver, count);
+ executorService.execute(fetchDataTask);
+
+ countDownLatch.await();
+
+ LOG.info(getClass().getSimpleName() + " load data finish, load data line size is " + count);
+ } catch (Exception e) {
+ LOG.error("failed loading data batch!", e);
+ } finally {
+ jdbcDriver.destroy();
+ }
+ }
+
+ protected class FetchDataTask implements Runnable {
+ long startIndex;
+ long endIndex;
+ String sql;
+ CountDownLatch countDownLatch;
+ JDBCDriver resource;
+ AtomicInteger finishedCount;//完成了多少条
+ int totalSize;//一共有多少条数据
+
+ public FetchDataTask(String sql, long startIndex, long endIndex, CountDownLatch countDownLatch,
+ AtomicInteger finishedCount, JDBCDriver resource, int totalSize) {
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ this.countDownLatch = countDownLatch;
+ this.sql = sql;
+ this.finishedCount = finishedCount;
+ this.resource = resource;
+ this.totalSize = totalSize;
+ }
+
+ @Override
+ public void run() {
+ long currentIndex = startIndex;
+ JSONObject msg = new JSONObject();
+ msg.put("endIndex", endIndex);
+ while (true) {
+ try {
+
+ msg.put("startIndex", currentIndex);
+
+ String sql = SQLUtil.parseIbatisSQL(msg, this.sql);
+ List<Map<String, Object>> rows = resource.queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ break;
+ }
+ currentIndex = Long.valueOf(rows.get(rows.size() - 1).get(idFieldName).toString());
+
+ int size = rows.size();
+ int count = finishedCount.addAndGet(size);
+ double progress = (double)count / (double)totalSize;
+ progress = progress * 100;
+ System.out.println(" finished count is " + count + " the total count is " + totalSize + ", the progress is " + String.format("%.2f", progress) + "%");
+ if (size < batchSize) {
+ if (size > 0) {
+
+ doProcess(rows);
+ }
+ break;
+ }
+ doProcess(rows);
+ } catch (Exception e) {
+ throw new RuntimeException("put data error ", e);
+ }
+ }
+
+ countDownLatch.countDown();
+ }
+ }
+
+ private void doProcess(List<Map<String, Object>> rows) {
+ for (Map<String, Object> row : rows) {
+ dataRowProcessor.doProcess(row);
+ }
+ }
+}
+
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/batchloader/IRowOperator.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/batchloader/IRowOperator.java
new file mode 100644
index 0000000..f67393f
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/batchloader/IRowOperator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver.batchloader;
+
+import java.util.Map;
+
+/**
+ * 操作一行数据
+ */
+public interface IRowOperator {
+
+ /**
+ * 处理一行数据
+ *
+ * @param row
+ */
+ void doProcess(Map<String, Object> row);
+
+}
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/orm/ORMUtil.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/orm/ORMUtil.java
new file mode 100644
index 0000000..20529b0
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/driver/orm/ORMUtil.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver.orm;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.model.Entity;
+import org.apache.rocketmq.streams.common.configurable.IFieldProcessor;
+import org.apache.rocketmq.streams.common.datatype.DataType;
+import org.apache.rocketmq.streams.common.metadata.MetaData;
+import org.apache.rocketmq.streams.common.utils.CollectionUtil;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.ReflectUtil;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+
+/**
+ * 轻量级的orm框架，如果pojo和table 符合驼峰的命名和下划线的命名规范，可以自动实现对象的orm
+ */
+public class ORMUtil {
+ private static final Log LOG = LogFactory.getLog(ORMUtil.class);
+
+ public static <T> T queryForObject(String sql, Object paras, Class<T> convertClass) {
+ return queryForObject(sql, paras, convertClass, null, null, null);
+ }
+
+ /**
+ * 通过sql查询一个唯一的对象出来，返回数据多于一条会报错
+ *
+ * @param sql 查询语句
+ * @param paras 如果有变参，这里面有变参的参数，可以是map，json或对象，只要key名或字段名和sql的参数名相同即可
+ * @param convertClass，如果有变参，这里面有变参的参数，可以是map，json或对象，只要key名或字段名和sql的参数名相同即可
+ * @param url 数据库连接url
+ * @param userName 用户名
+ * @param password 密码
+ * @param <T>
+ * @return 转换后的对象
+ */
+ public static <T> T queryForObject(String sql, Object paras, Class<T> convertClass, String url, String userName,
+ String password) {
+ List<T> result = queryForList(sql, paras, convertClass, url, userName, password);
+ if (result == null || result.size() == 0) {
+ return null;
+ }
+ if (result.size() > 1) {
+ throw new RuntimeException("expect only one row, actual " + result.size() + ". the builder is " + sql);
+ }
+ return result.get(0);
+ }
+
+ /**
+ * 根据sql查询一批对象
+ *
+ * @param sql 查询语句
+ * @param paras 如果有变参，这里面有变参的参数，可以是map，json或对象，只要key名或字段名和sql的参数名相同即可
+ * @param convertClass 如果有变参，这里面有变参的参数，可以是map，json或对象，只要key名或字段名和sql的参数名相同即可
+ * @param <T>
+ * @return 返回对象列表
+ */
+ public static <T> List<T> queryForList(String sql, Object paras, Class<T> convertClass) {
+ return queryForList(sql, paras, convertClass, null, null, null);
+ }
+
+ public static boolean hasConfigueDB() {
+ return ComponentCreator.getProperties().getProperty(ConfigureFileKey.JDBC_URL) != null;
+ }
+
+ /**
+ * @param sql 查询语句
+ * @param paras 如果有变参，这里面有变参的参数，可以是map，json或对象，只要key名或字段名和sql的参数名相同即可
+ * @param convertClass 需要转换成的对象类。类的字段应该符合列名的命名规范，下划线换成驼峰形式
+ * @param url 数据库连接url
+ * @param userName 用户名
+ * @param password 密码
+ * @param <T>
+ * @return 返回对象列表
+ */
+ public static <T> List<T> queryForList(String sql, Object paras, Class<T> convertClass, String url, String userName,
+ String password) {
+ sql = SQLUtil.parseIbatisSQL(paras, sql);
+ JDBCDriver dataSource = null;
+ try {
+ if (StringUtil.isEmpty(url) || StringUtil.isEmpty(userName)) {
+ dataSource = DriverBuilder.createDriver();
+ } else {
+ dataSource = DriverBuilder.createDriver(DriverBuilder.DEFALUT_JDBC_DRIVER, url, userName, password);
+ }
+
+ List<Map<String, Object>> rows = dataSource.queryForList(sql);
+ List<T> result = new ArrayList();
+ for (Map<String, Object> row : rows) {
+ T t = convert(row, convertClass);
+ result.add(t);
+ }
+ return result;
+ } catch (Exception e) {
+ String errorMsg = ("query for list error ,the builder is " + sql + ". the error msg is " + e.getMessage());
+ LOG.error(errorMsg);
+ e.printStackTrace();
+ throw new RuntimeException(errorMsg, e);
+ } finally {
+ if (dataSource != null) {
+ dataSource.destroy();
+ }
+ }
+ }
+
+ /**
+ * 执行sql，sql中可以有mybatis的参数#{name}
+ *
+ * @param sql insert语句
+ * @param paras 可以是map，json或对象，只要key名或字段名和sql的参数名相同即可
+ * @return
+ */
+ public static boolean executeSQL(String sql, Object paras) {
+ if (paras != null) {
+ sql = SQLUtil.parseIbatisSQL(paras, sql);
+ }
+ JDBCDriver dataSource = null;
+ try {
+ dataSource = DriverBuilder.createDriver();
+ dataSource.execute(sql);
+ return true;
+ } catch (Exception e) {
+ String errorMsg = ("execute sql error ,the sql is " + sql + ". the error msg is " + e.getMessage());
+ LOG.error(errorMsg);
+ e.printStackTrace();
+ throw new RuntimeException(errorMsg, e);
+ } finally {
+ if (dataSource != null) {
+ dataSource.destroy();
+ }
+ }
+ }
+
+ /**
+ * 把一个对象的字段拼接成where条件，如果字段值为null，不拼接
+ *
+ * @param object 带拼接的对象
+ * @param fieldNames 需要拼接的字段名，如果为null，返回null
+ * @return where 部分的sql
+ */
+ public static String createQueryWhereSql(Object object, String... fieldNames) {
+ if (fieldNames == null || fieldNames.length == 0) {
+ return "";
+ }
+ StringBuilder stringBuilder = new StringBuilder();
+ boolean isFirst = true;
+ for (String fieldName : fieldNames) {
+ Object value = ReflectUtil.getBeanFieldValue(object, fieldName);
+ if (object != null && value == null) {
+ continue;
+ }
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ stringBuilder.append(" and ");
+ }
+ String columnName = getColumnNameFromFieldName(fieldName);
+ stringBuilder.append(" " + columnName + "=#{" + fieldName + "} ");
+ }
+ return stringBuilder.toString();
+ }
+
+ /**
+ * 把一行数据转换成一个对象，符合驼峰的命名和下划线的命名规范
+ *
+ * @param row 一行数据
+ * @param clazz 待转化的对象类型
+ * @param <T>
+ * @return 转化对象
+ */
+ public static <T> T convert(Map<String, Object> row, Class<T> clazz) {
+ T t = ReflectUtil.forInstance(clazz);
+ Iterator<Map.Entry<String, Object>> it = row.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, Object> entry = it.next();
+ String columnName = entry.getKey();
+ Object value = entry.getValue();
+ if (value == null) {
+ continue;
+ }
+ String fieldName = getFieldNameFromColumnName(columnName);
+ DataType datatype = DataTypeUtil.createFieldDataType(clazz, fieldName);
+ Object columnValue = datatype.convert(value);
+ ReflectUtil.setBeanFieldValue(t, fieldName, columnValue);
+ }
+ return t;
+ }
+
+ /**
+ * 把列名转换成字段名称，把下划线转化成驼峰
+ *
+ * @param columnName
+ * @return
+ */
+ protected static String getFieldNameFromColumnName(String columnName) {
+ String[] values = columnName.split("_");
+ if (values.length == 1) {
+ return columnName;
+ }
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(values[0]);
+ for (int i = 1; i < values.length; i++) {
+ String value = values[i];
+ value = value.substring(0, 1).toUpperCase() + value.substring(1);
+ stringBuilder.append(value);
+ }
+ return stringBuilder.toString();
+ }
+
+ /**
+ * 对象批量替换，会生成replace into语句，多个对象会拼接成一个sql，提升效率
+ *
+ * @param values 待插入对象
+ */
+ public static void batchReplaceInto(Collection values) {
+ List list = new ArrayList<>();
+ list.addAll(values);
+ batchReplaceInto(list);
+ }
+
+ public static void batchReplaceInto(Object... valueArray) {
+ List values = new ArrayList();
+ if (valueArray != null) {
+ for (Object value : valueArray) {
+ values.add(value);
+ }
+ }
+ batchReplaceInto(values);
+ }
+
+ public static void batchReplaceInto(List<?> values) {
+ batchIntoByFlag(values, 1);
+ }
+
+ /**
+ * 对象批量插入，如果主键冲突会忽略，会生成insert ignore into语句，多个对象会拼接成一个sql，提升效率
+ *
+ * @param values 待插入对象
+ */
+ public static void batchIgnoreInto(List<?> values) {
+ batchIntoByFlag(values, -1);
+ }
+
+ /**
+ * 对象批量插入，如果主键冲突会忽略，会生成insert ignore into语句，多个对象会拼接成一个sql，提升效率
+ *
+ * @param values 待插入对象
+ */
+ public static void batchInsertInto(List<?> values) {
+ batchIntoByFlag(values, 0);
+ }
+
+ /**
+ * 批量插入对象，多个对象会拼接成一个sql flag==1 then replace into flag=-1 then insert ignore int flag=0 then insert int
+ *
+ * @param values
+ * @param flag
+ */
+ protected static void batchIntoByFlag(List<?> values, int flag) {
+ if (CollectionUtil.isEmpty(values)) {
+ return;
+ }
+ Object object = values.get(0);
+ Map<String, Object> paras = new HashMap<>(16);
+ MetaData metaData = createMetaDate(object, paras);
+ boolean containsIdField = false;
+ if (metaData.getIdFieldName() != null) {
+ for (Object o : values) {
+ Object id = ReflectUtil.getDeclaredField(o, metaData.getIdFieldName());
+ if (id == null) {
+ containsIdField = false;
+ break;
+ }
+ if (id instanceof Number) {
+ if (Long.valueOf(id.toString()) == 0) {
+ containsIdField = false;
+ break;
+ }
+ }
+ if (id instanceof String) {
+ String idStr = (String)id;
+ if (StringUtil.isEmpty(idStr)) {
+ containsIdField = false;
+ break;
+ }
+ }
+ }
+ }
+
+ String sql = null;
+ if (flag == 0) {
+ sql = SQLUtil.createInsertSql(metaData, paras, containsIdField);
+ } else if (flag == 1) {
+ sql = SQLUtil.createInsertSql(metaData, paras, containsIdField);
+ } else if (flag == -1) {
+ sql = SQLUtil.createIgnoreInsertSql(metaData, paras, containsIdField);
+ } else {
+ throw new RuntimeException("the flag is not valdate " + flag);
+ }
+
+ List<Map<String, Object>> rows = new ArrayList<>();
+ for (int i = 1; i < values.size(); i++) {
+ Map<String, Object> row = createRow(metaData, values.get(i));
+ rows.add(row);
+ }
+ String valuesSQL = SQLUtil.createInsertValuesSQL(metaData, rows);
+ sql = sql + valuesSQL + " ON DUPLICATE KEY UPDATE " + SQLUtil.createDuplicateKeyUpdateSQL(metaData);
+ ;
+ JDBCDriver dataSource = DriverBuilder.createDriver();
+ try {
+ dataSource.execute(sql);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ } finally {
+ if (dataSource != null) {
+ dataSource.destroy();
+ }
+ }
+ }
+
+ private static Map<String, Object> createRow(MetaData metaData, Object object) {
+ Map<String, Object> row = new HashMap<>();
+ ReflectUtil.scanFields(object, new IFieldProcessor() {
+ @Override
+ public void doProcess(Object o, Field field) {
+ String fieldName = field.getName();
+ String columnName = getColumnNameFromFieldName(fieldName);
+ Object value = ReflectUtil.getBeanFieldValue(o, fieldName);
+ row.put(columnName, value);
+
+ }
+ });
+ return row;
+ }
+
+ public static void replaceInto(Object object) {
+ replaceInto(object, null, null, null);
+ }
+
+ /**
+ * 把一个对象插入到数据库，对象符合插入规范，表名是对象名转小写后加下划线。如果有重复到会被替换成最新的
+ *
+ * @param object
+ * @param url
+ * @param userName
+ * @param password
+ */
+ public static void replaceInto(Object object, String url, String userName, String password) {
+ Map<String, Object> paras = new HashMap<>();
+ if (Entity.class.isInstance(object)) {
+ Entity newEntity = (Entity)object;
+ newEntity.setGmtModified(new Date());
+ if (newEntity.getGmtCreate() == null) {
+ newEntity.setGmtCreate(new Date());
+ }
+ }
+ MetaData metaData = createMetaDate(object, paras);
+ String sql = SQLUtil.createReplacesInsertSql(metaData, paras, metaData.getIdFieldName() != null);
+ JDBCDriver dataSource = null;
+ try {
+ if (StringUtil.isEmpty(url) || StringUtil.isEmpty(userName)) {
+ dataSource = DriverBuilder.createDriver();
+ } else {
+ dataSource = DriverBuilder.createDriver(DriverBuilder.DEFALUT_JDBC_DRIVER, url, userName, password);
+ }
+ long id = dataSource.executeInsert(sql);
+ if (Entity.class.isInstance(object)) {
+ Entity newEntity = (Entity)object;
+ newEntity.setId(id);
+ }
+ } catch (Exception e) {
+ String errorMsg = ("replace into error ,the builder is " + sql + ". the error msg is " + e.getMessage());
+ LOG.error(errorMsg);
+ throw new RuntimeException(errorMsg, e);
+ } finally {
+ if (dataSource != null) {
+ dataSource.destroy();
+ }
+ }
+ }
+
+ public static void insertInto(Object object, boolean ignoreRepeateRow) {
+ insertInto(object, ignoreRepeateRow, null, null, null);
+ }
+
+ /**
+ * 把一个对象插入到数据库，对象符合插入规范，表名是对象名转小写后加下划线
+ *
+ * @param object
+ * @param ignoreRepeateRow，如果是重复数据，则不插入。基于唯一建做判断
+ * @param url
+ * @param userName
+ * @param password
+ */
+ public static void insertInto(Object object, boolean ignoreRepeateRow, String url, String userName,
+ String password) {
+ Map<String, Object> paras = new HashMap<>();
+ if (Entity.class.isInstance(object)) {
+ Entity newEntity = (Entity)object;
+ newEntity.setGmtCreate(DateUtil.getCurrentTime());
+ newEntity.setGmtModified(DateUtil.getCurrentTime());
+ }
+ MetaData metaData = createMetaDate(object, paras);
+ String sql = null;
+ if (ignoreRepeateRow) {
+ sql = SQLUtil.createIgnoreInsertSql(metaData, paras, metaData.getIdFieldName() != null);
+ } else {
+ sql = SQLUtil.createInsertSql(metaData, paras, metaData.getIdFieldName() != null);
+ }
+ JDBCDriver dataSource = null;
+ try {
+ if (StringUtil.isEmpty(url) || StringUtil.isEmpty(userName)) {
+ dataSource = DriverBuilder.createDriver();
+ } else {
+ dataSource = DriverBuilder.createDriver(DriverBuilder.DEFALUT_JDBC_DRIVER, url, userName, password);
+ }
+ long id = dataSource.executeInsert(sql);
+ if (Entity.class.isInstance(object)) {
+ Entity newEntity = (Entity)object;
+ newEntity.setId(id);
+ }
+ } catch (Exception e) {
+ String errorMsg = ("insert into error ,the builder is " + sql + ". the error msg is " + e.getMessage());
+ LOG.error(errorMsg);
+ throw new RuntimeException(errorMsg, e);
+ } finally {
+ if (dataSource != null) {
+ dataSource.destroy();
+ }
+ }
+ }
+
+ /**
+ * 创建meta信息
+ *
+ * @param object
+ * @param paras
+ * @return
+ */
+ public static MetaData createMetaDate(Object object, Map<String, Object> paras) {
+ MetaData metaData = MetaData.createMetaDate(object, paras);
+ return metaData;
+ }
+
+ public static String getTableName(Class clazz) {
+ return getColumnNameFromFieldName(clazz.getSimpleName());
+ }
+
+ /**
+ * 把驼峰转换成下划线的形式
+ *
+ * @param para
+ * @return
+ */
+ protected static String getColumnNameFromFieldName(String para) {
+ return MetaData.getColumnNameFromFieldName(para);
+ }
+
+}
diff --git a/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/operator/SQLOperator.java b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/operator/SQLOperator.java
new file mode 100644
index 0000000..1f42470
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/main/java/org/apache/rocketmq/streams/db/operator/SQLOperator.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.operator;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.Changeable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.interfaces.IStreamOperator;
+import org.apache.rocketmq.streams.common.context.AbstractContext;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.IStageBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.NewSQLChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * sql算法，执行一个sql，sql中可以有变量，会用message的值做替换。
+ */
+public class SQLOperator extends BasedConfigurable implements IStreamOperator<IMessage, IMessage>, IStageBuilder<ChainStage> {
+ private static final Log LOG = LogFactory.getLog(SQLOperator.class);
+ public static final String DEFALUT_DATA_KEY = "data";
+
+ @ENVDependence
+ protected String jdbcDriver = AbstractComponent.DEFAULT_JDBC_DRIVER;
+ @ENVDependence
+ protected String url;
+ @ENVDependence
+ protected String userName;
+ @ENVDependence
+ protected String password;
+
+ @Changeable
+ protected String sql;//查询的sql，支持ibatis的语法和变量.因为会被替换，所以不自动感知。select * from table where name=#{name=chris}
+
+ public SQLOperator() {
+ setType(IStreamOperator.TYPE);
+ }
+
+ public SQLOperator(String sql, String url, String userName, String password) {
+ this();
+ this.sql = sql;
+ this.url = url;
+ this.password = password;
+ this.userName = userName;
+ }
+
+ /**
+ * db串多数是名字，可以取个名字前缀，如果值为空，默认为此类的name，name为空，默认为简单类名
+ *
+ * @param sql
+ * @param dbInfoNamePrex
+ */
+ public SQLOperator(String sql, String dbInfoNamePrex) {
+ this();
+ if (StringUtil.isEmpty(dbInfoNamePrex)) {
+ dbInfoNamePrex = getConfigureName();
+ }
+ if (StringUtil.isEmpty(dbInfoNamePrex)) {
+ dbInfoNamePrex = this.getClass().getSimpleName();
+ }
+ this.sql = sql;
+ this.url = dbInfoNamePrex + ".url";
+ this.password = dbInfoNamePrex + ".password";
+ ;
+ this.userName = dbInfoNamePrex + ".userName";
+ }
+
+ @Override
+ public IMessage doMessage(IMessage message, AbstractContext context) {
+ String querySQL = SQLUtil.parseIbatisSQL(message.getMessageBody(), sql);
+ List<Map<String, Object>> result = query(querySQL);
+ message.getMessageBody().put(DEFALUT_DATA_KEY, result);
+ return message;
+ }
+
+ /**
+ * 查询数据库数据
+ *
+ * @return
+ */
+ protected List<Map<String, Object>> query(String querySQL) {
+
+ JDBCDriver dataSource = null;
+ try {
+ dataSource = createDBDataSource();
+ List<Map<String, Object>> result = null;
+ result = dataSource.queryForList(sql);
+
+ return result;
+ } finally {
+ if (dataSource != null) {
+ dataSource.destroy();
+ }
+ }
+
+ }
+
+ public JDBCDriver createDBDataSource() {
+ return DriverBuilder.createDriver(jdbcDriver, url, userName, password);
+ }
+
+ @Override
+ public void addConfigurables(PipelineBuilder pipelineBuilder) {
+ pipelineBuilder.addConfigurables(this);
+ }
+
+ @Override
+ public ChainStage createStageChain(PipelineBuilder pipelineBuilder) {
+ NewSQLChainStage sqlChainStage = new NewSQLChainStage();
+ sqlChainStage.setMessageProcessor(this);
+ return sqlChainStage;
+ }
+
+ public String getSql() {
+ return sql;
+ }
+
+ public void setSql(String sql) {
+ this.sql = sql;
+ }
+
+ public String getJdbcDriver() {
+ return jdbcDriver;
+ }
+
+ public void setJdbcDriver(String jdbcDriver) {
+ this.jdbcDriver = jdbcDriver;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+}
diff --git a/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/Person.java b/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/Person.java
new file mode 100644
index 0000000..3d5e51b
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/Person.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+
+public class Person extends BasedConfigurable {
+ @ENVDependence
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+
+ @Override
+ public Object clone() {
+ Person person = null;
+ try {
+ person = (Person)super.clone();
+ } catch (CloneNotSupportedException e) {
+ System.out.println("clone error " + e);
+ }
+ return person;
+ }
+}
diff --git a/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureServiceTest.java b/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureServiceTest.java
new file mode 100644
index 0000000..3baa65d
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/configuable/DBSupportParentConfigureServiceTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.configuable;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.configurable.model.Configure;
+import org.apache.rocketmq.streams.db.Person;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * 数据库的存储，需要配置存储的连接参数，请先完成配置，后执行单元用例 如果未建表，可以通过Configure.createTableSQL() 获取建表语句，创建表后，测试
+ */
+public class DBSupportParentConfigureServiceTest {
+ private String URL = "";
+ protected String USER_NAME = "";
+ protected String PASSWORD = "";
+ protected String TABLE_NAME = "dipper_configure_source";
+
+ @Test
+ public void testDBConfigurableService() {
+ String namespace = "streams.db.configuable";
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE, "DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_TABLE_NAME, TABLE_NAME);
+
+ //如果表不存在，创建表
+ String sql = (Configure.createTableSQL(TABLE_NAME));
+ DriverBuilder.createDriver().execute(sql);
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ configurableComponent.insert(createPerson(namespace));
+ configurableComponent.refreshConfigurable(namespace);
+ Person person = configurableComponent.queryConfigurable("person", "peronName");
+ assertTrue(person != null);
+ }
+
+ /**
+ * 创建configuable对象
+ *
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setName("chris");
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
diff --git a/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/driver/orm/ORMUtilTest.java b/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/driver/orm/ORMUtilTest.java
new file mode 100644
index 0000000..e9b1fa2
--- /dev/null
+++ b/rocketmq-streams-db-operator/src/test/java/org/apache/rocketmq/streams/db/driver/orm/ORMUtilTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.driver.orm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.db.Person;
+import org.junit.Test;
+
+public class ORMUtilTest {
+ private String URL = "";
+ protected String USER_NAME = "";
+ protected String PASSWORD = "";
+
+ public ORMUtilTest() {
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
+ }
+
+ @Test
+ public void testInsert() {
+ String namespace = "org.apache.configuable.test";
+ List<Person> personList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ personList.add(createPerson(namespace, "chris" + i));
+ }
+ /**
+ * 不带数据库连接信息（url,userName,Password），使用ConfiguableComponet的连接信息
+ */
+ ORMUtil.batchIgnoreInto(personList);//批量插入，如果有唯一键冲突，替换
+ ORMUtil.batchIgnoreInto(personList);//批量插入，如果有唯一键冲突，忽略
+ ORMUtil.batchIntoByFlag(personList, 0);////批量插入，如果有唯一键冲突，跑错
+ }
+
+ @Test
+ public void testQueryList() {
+ Map<String, Integer> paras = new HashMap<>();
+ paras.put("age", 18);
+ List<Person> personList = ORMUtil.queryForList("select * from person where age >${age} limit 100", paras, Person.class);
+ }
+
+ @Test
+ public void testQueryOneRow() {
+ Person personPara = new Person();
+ personPara.setAge(18);
+ personPara.setName("chris1");
+ Person person = ORMUtil.queryForObject("select * from person where age =${age} and name='${name}' ", personPara, Person.class, URL, USER_NAME, PASSWORD);
+ }
+
+ /**
+ * 创建configuable对象
+ *
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace, String name) {
+ Person person = new Person();
+ person.setName(name);
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
diff --git a/rocketmq-streams-transport-minio/pom.xml b/rocketmq-streams-transport-minio/pom.xml
new file mode 100755
index 0000000..510b8cc
--- /dev/null
+++ b/rocketmq-streams-transport-minio/pom.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-transport-minio</artifactId>
+ <packaging>jar</packaging>
+ <name>ROCKETMQ STREAMS :: transport-minio</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-serviceloader</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-transport-minio/rocketmq-streams-transport-minio.iml b/rocketmq-streams-transport-minio/rocketmq-streams-transport-minio.iml
new file mode 100644
index 0000000..af11f77
--- /dev/null
+++ b/rocketmq-streams-transport-minio/rocketmq-streams-transport-minio.iml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+ <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_5">
+ <output url="file://$MODULE_DIR$/${project.build.directory}/classes" />
+ <output-test url="file://$MODULE_DIR$/${project.build.directory}/test-classes" />
+ <content url="file://$MODULE_DIR$">
+ <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
+ <excludeFolder url="file://$MODULE_DIR$/${project.build.directory}/classes" />
+ <excludeFolder url="file://$MODULE_DIR$/${project.build.directory}/test-classes" />
+ <excludeFolder url="file://$MODULE_DIR$/target" />
+ </content>
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ </component>
+</module>
\ No newline at end of file
diff --git a/rocketmq-streams-transport-minio/src/main/java/org/apache/rocketmq/streams/transport/minio/MinioFileTransport.java b/rocketmq-streams-transport-minio/src/main/java/org/apache/rocketmq/streams/transport/minio/MinioFileTransport.java
new file mode 100644
index 0000000..c2a6884
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/main/java/org/apache/rocketmq/streams/transport/minio/MinioFileTransport.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.transport.minio;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import org.apache.rocketmq.streams.common.transport.AbstractFileTransport;
+import org.apache.rocketmq.streams.common.transport.IFileTransport;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import com.google.auto.service.AutoService;
+import io.minio.MinioClient;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+@AutoService(IFileTransport.class)
+@ServiceName(MinioFileTransport.NAME)
+public class MinioFileTransport extends AbstractFileTransport {
+ public static final String NAME = "minio";
+ protected String ak;
+ protected String sk;
+ protected String endpoint;
+ protected String dirpperDir;
+ protected MinioClient minioClient;
+
+ public MinioFileTransport() {
+ this.ak = ComponentCreator.getProperties().getProperty(ConfigureFileKey.FILE_TRANSPORT_AK);
+ this.sk = ComponentCreator.getProperties().getProperty(ConfigureFileKey.FILE_TRANSPORT_SK);
+ this.endpoint = ComponentCreator.getProperties().getProperty(ConfigureFileKey.FILE_TRANSPORT_ENDPOINT);
+ this.dirpperDir = ComponentCreator.getProperties().getProperty(ConfigureFileKey.FILE_TRANSPORT_DIPPER_DIR);
+ if (StringUtil.isEmpty(this.dirpperDir)) {
+ this.dirpperDir = "dipper_files";
+ }
+ }
+
+ @Override
+ public File download(String remoteFileName, String localDir, String localFileName) {
+ MinioClient minioClient = getOrCreateMinioClient();
+ BufferedWriter bw = null;
+ BufferedReader br = null;
+ try {
+ InputStream input = minioClient.getObject(dirpperDir, remoteFileName);
+ File file = new File(FileUtil.concatFilePath(localDir, localFileName));
+ bw = new BufferedWriter(new FileWriter(file));
+ br = new BufferedReader(new InputStreamReader(input));
+ String line = br.readLine();
+ while (line != null) {
+ bw.write(line);
+ line = br.readLine();
+ }
+ bw.flush();
+ return file;
+ } catch (Exception e) {
+ throw new RuntimeException("dowload file error " + dirpperDir + "/" + remoteFileName, e);
+
+ } finally {
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ if (br != null) {
+ try {
+ br.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ }
+
+ @Override
+ public Boolean upload(File file, String remoteFileName) {
+ MinioClient minioClient = getOrCreateMinioClient();
+ try {
+ minioClient.putObject(dirpperDir, remoteFileName, file.getAbsolutePath());
+ } catch (Exception e) {
+ throw new RuntimeException("upload file error " + dirpperDir + "/" + remoteFileName, e);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean delete(String remoteFileName) {
+ MinioClient minioClient = getOrCreateMinioClient();
+ try {
+ minioClient.removeObject(dirpperDir, remoteFileName);
+ } catch (Exception e) {
+ throw new RuntimeException("delete file error " + dirpperDir + "/" + remoteFileName, e);
+ }
+ return true;
+ }
+
+ protected MinioClient getOrCreateMinioClient() {
+ if (this.minioClient == null) {
+ synchronized (this) {
+ if (minioClient == null) {
+ try {
+ MinioClient minioClient = new MinioClient(endpoint, ak, sk);
+ boolean existDir = minioClient.bucketExists(dirpperDir);
+
+ if (!existDir) {
+ minioClient.makeBucket(dirpperDir);
+ }
+ this.minioClient = minioClient;
+ } catch (Exception e) {
+ throw new RuntimeException("create minio client error", e);
+ }
+
+ }
+ }
+ }
+ return this.minioClient;
+ }
+}
+
+
diff --git a/rocketmq-streams-transport-minio/src/test/java/com/aliyun/yundun/dipper/configurable/DataTpyeTest.java b/rocketmq-streams-transport-minio/src/test/java/com/aliyun/yundun/dipper/configurable/DataTpyeTest.java
new file mode 100644
index 0000000..a3ebf6b
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/java/com/aliyun/yundun/dipper/configurable/DataTpyeTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.yundun.dipper.configurable;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.configurable.model.Person;
+
+public class DataTpyeTest {
+ @Test
+ public void testDataType() {
+ Person person = Person.createPerson("com.dipper.test");
+ String jsonValue = person.toJson();
+
+ Person person1 = new Person();
+ person1.toObject(jsonValue);
+ System.out.println(person1);
+ }
+
+ @Test
+ public void testV2() {
+ Set<String> set = new HashSet<>();
+ set.add("北斗");
+ set.add("福建jz");
+ set.add("甘肃jz");
+ set.add("广东省气象micaps云");
+ set.add("贵州公安科信");
+ set.add("贵州警务云");
+ set.add("杭州税友");
+ set.add("江西公安大数据平台");
+ set.add("昆仑项目");
+ set.add("新华网");
+ set.add("浙江气象高时空分辨率气象预报专有云");
+ List<String> v2 = FileUtil.loadFileLine("/Users/yuanxiaodong/Documents/workdir/项目名称.txt");
+ List<String> zyy = FileUtil.loadFileLine("/Users/yuanxiaodong/Documents/workdir/专有云.txt");
+ int count = 0;
+ for (String v2Line : v2) {
+ boolean match = false;
+ for (String zyyLine : zyy) {
+ if (zyyLine.indexOf(v2Line) != -1 || v2Line.indexOf(zyyLine) != -1) {
+ match = true;
+ count++;
+ }
+ }
+ if (match == false) {
+ System.out.println(v2Line);
+ }
+ }
+ System.out.println(count);
+ }
+}
diff --git a/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configuable/model/DataTpyeTest.java b/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configuable/model/DataTpyeTest.java
new file mode 100644
index 0000000..a1570c3
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configuable/model/DataTpyeTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.model;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.junit.Test;
+
+public class DataTpyeTest {
+ @Test
+ public void testDataType() {
+ Person person = Person.createPerson("com.dipper.test");
+ String jsonValue = person.toJson();
+
+ Person person1 = new Person();
+ person1.toObject(jsonValue);
+ System.out.println(person1);
+ }
+
+ @Test
+ public void testV2() {
+ Set<String> set = new HashSet<>();
+ set.add("北斗");
+ set.add("福建jz");
+ set.add("甘肃jz");
+ set.add("广东省气象micaps云");
+ set.add("贵州公安科信");
+ set.add("贵州警务云");
+ set.add("杭州税友");
+ set.add("江西公安大数据平台");
+ set.add("昆仑项目");
+ set.add("新华网");
+ set.add("浙江气象高时空分辨率气象预报专有云");
+ List<String> v2 = FileUtil.loadFileLine("/Users/yuanxiaodong/Documents/workdir/项目名称.txt");
+ List<String> zyy = FileUtil.loadFileLine("/Users/yuanxiaodong/Documents/workdir/专有云.txt");
+ int count = 0;
+ for (String v2Line : v2) {
+ boolean match = false;
+ for (String zyyLine : zyy) {
+ if (zyyLine.indexOf(v2Line) != -1 || v2Line.indexOf(zyyLine) != -1) {
+ match = true;
+ count++;
+ }
+ }
+ if (match == false) {
+ System.out.println(v2Line);
+ }
+ }
+ System.out.println(count);
+ }
+}
diff --git a/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configuable/model/Person.java b/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configuable/model/Person.java
new file mode 100644
index 0000000..04b99bb
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configuable/model/Person.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.configuable.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+
+public class Person extends BasedConfigurable {
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+}
diff --git a/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configurable/model/Person.java b/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configurable/model/Person.java
new file mode 100644
index 0000000..709978a
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/java/org/apache/rocketmq/streams/configurable/model/Person.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.configurable.model;
+
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Person extends BasedConfigurable {
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+}
diff --git a/rocketmq-streams-transport-minio/src/test/resources/component/ConfigurableComponent.properties b/rocketmq-streams-transport-minio/src/test/resources/component/ConfigurableComponent.properties
new file mode 100644
index 0000000..598511e
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/resources/component/ConfigurableComponent.properties
@@ -0,0 +1,7 @@
+dipper.configurable.service.type=resource_support_parent
+dipper.channle.ak=xxxxxx
+dipper.channle.sk=xxxxxx
+dipper.rds.jdbc.driver=com.mysql.jdbc.Driver
+dipper.rds.jdbc.url=xxxxxxx
+dipper.rds.jdbc.username=xxxxxx
+dipper.rds.jdbc.password=xxxxx
\ No newline at end of file
diff --git a/rocketmq-streams-transport-minio/src/test/resources/log4j.xml b/rocketmq-streams-transport-minio/src/test/resources/log4j.xml
new file mode 100755
index 0000000..7812fe7
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/resources/log4j.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
diff --git a/rocketmq-streams-transport-minio/src/test/resources/pro-function.txt b/rocketmq-streams-transport-minio/src/test/resources/pro-function.txt
new file mode 100644
index 0000000..34a186f
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/resources/pro-function.txt
@@ -0,0 +1,11 @@
+paserBySplit(@,uuid,file_path,pid,ppid,pfile_path,group_name,group_id,user_name,uid,euid,egroup_id,time,cmd_line,index,perm,tty,pcmdline,sid,cwd,filename);
+addRandom(messageId,10);
+rename(groupname,group_name);
+rename(username,user_name);
+rename(seq,index);
+rename(egourpid,egroup_id);
+rename(filepath,file_path);
+rename(groupid,group_id);
+rename(pfilename,pfile_name);
+rename(safe_mode,perm);
+rename(cmdline,cmd_line);
\ No newline at end of file
diff --git a/rocketmq-streams-transport-minio/src/test/resources/python_script.py b/rocketmq-streams-transport-minio/src/test/resources/python_script.py
new file mode 100644
index 0000000..f4e7252
--- /dev/null
+++ b/rocketmq-streams-transport-minio/src/test/resources/python_script.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# coding=utf-8
+import json;
+import re;
+import time;
+regex = '^/(.*)/\w+'
+pattern = re.compile(regex)
+
+def pythonTest(*processLine):
+ try:
+ jsonObject = json.loads(processLine[0])
+
+ if jsonObject.has_key('filepath'):
+ filePath = jsonObject['filepath']
+ match = pattern.search(filePath)
+ if match:
+ return match.group(1)
+ else:
+ pass # print "does not has key filepath"
+ except BaseException as e:
+ pass # print "process one line cause exception %s" %e
+ return "does not match"
[rocketmq-streams] 10/15: modify the README.md
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit a9450c971b9a4905ec52ac6c379564d929ecdda8
Author: 刈刀 <je...@gmail.com>
AuthorDate: Mon Aug 2 12:25:35 2021 +0800
modify the README.md
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 649319c..3b163db 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
## Features
+
* 轻量级部署：可以单独部署，也支持集群部署
* 多种类型的数据输入以及输出，source支持 rocketmq ， sink支持db, rocketmq 等
[rocketmq-streams] 12/15: modify the README.md
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 5555b67eba44b45356a53082ec8b7b496b5fbf9e
Author: 刈刀 <je...@gmail.com>
AuthorDate: Mon Aug 2 12:48:41 2021 +0800
modify the README.md
---
README.md | 30 ++++++++++++++++--------------
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git a/README.md b/README.md
index 649319c..92a9b2f 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,7 @@ StreamBuilder 用于构建流任务的源； 内部包含```dataStream()```和``
## DataStream API
### Source
+
DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据；
+ ```fromFile``` 从文件中读取数据， 该方法包含俩个参数
@@ -56,13 +57,15 @@ DataStreamSource 是分段式编程的源头类，用于对接各种数据源，
+ ```groupName``` 消费者组的名称，必填参数
+ ```isJson``` 是否json格式，非必填参数
+ ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
-
+
+ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
### transform
+
transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
#### DataStream
+
DataStream实现了一系列常见的流计算算子
+ ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream
@@ -76,30 +79,29 @@ DataStream实现了一系列常见的流计算算子
+ ```toFile``` 将结果保存为文件，生成一个新的DataStreamAction实例
+ ```toDB``` 将结果保存到数据库
+ ```toRocketmq``` 将结果输出到rocketmq
-+ ```toSls``` 将结果输出到sls
+ ```to``` 将结果经过自定义的ISink接口输出到指定的存储
+ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
- + ```count``` 在窗口内计数
- + ```min``` 获取窗口内统计值的最小值
- + ```max``` 获取窗口内统计值得最大值
- + ```avg``` 获取窗口内统计值的平均值
- + ```sum``` 获取窗口内统计值的加和值
- + ```reduce``` 在窗口内进行自定义的汇总运算
+ + ```count``` 在窗口内计数
+ + ```min``` 获取窗口内统计值的最小值
+ + ```max``` 获取窗口内统计值得最大值
+ + ```avg``` 获取窗口内统计值的平均值
+ + ```sum``` 获取窗口内统计值的加和值
+ + ```reduce``` 在窗口内进行自定义的汇总运算
+ ```join``` 根据条件将将俩个流进行关联， 合并为一个大流进行相关的运算
+ ```union``` 将俩个流进行合并
+ ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算
+ ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等
+# Strategy
-# Strategy
策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
```java
//指定checkpoint的存储策略
source
-.fromRocketmq("TSG_META_INFO", "")
-.map(message -> message + "--")
-.toPrint(1)
-.with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX", "", "", 0L))
-.start();
+ .fromRocketmq("TSG_META_INFO","")
+ .map(message->message+"--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
+ .start();
```
\ No newline at end of file
[rocketmq-streams] 11/15: modify the README.md
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit ecb842a0a5930488d74bbc9cd906a1306571c0a8
Author: 刈刀 <je...@gmail.com>
AuthorDate: Mon Aug 2 12:35:26 2021 +0800
modify the README.md
---
README.md | 1 -
1 file changed, 1 deletion(-)
diff --git a/README.md b/README.md
index 3b163db..649319c 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,6 @@
## Features
-
* 轻量级部署：可以单独部署，也支持集群部署
* 多种类型的数据输入以及输出，source支持 rocketmq ， sink支持db, rocketmq 等
[rocketmq-streams] 06/15: modify the README.md
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 30cb39bdbda5ac76d2b350867619ae7d70212979
Author: 刈刀 <ju...@alibaba-inc.com>
AuthorDate: Mon Aug 2 12:12:29 2021 +0800
modify the README.md
---
README.md | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 2dfb623..649319c 100644
--- a/README.md
+++ b/README.md
@@ -56,8 +56,7 @@ DataStreamSource 是分段式编程的源头类，用于对接各种数据源，
+ ```groupName``` 消费者组的名称，必填参数
+ ```isJson``` 是否json格式，非必填参数
+ ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
-
-
+
+ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
### transform
[rocketmq-streams] 09/15: modify the README.md
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit ec252775987b1acbeb1fb848a819ccf176e7fae4
Author: 刈刀 <je...@gmail.com>
AuthorDate: Mon Aug 2 12:23:37 2021 +0800
modify the README.md
---
README.md | 1 -
1 file changed, 1 deletion(-)
diff --git a/README.md b/README.md
index 5325cdd..649319c 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,6 @@
## DataStream Example
-
```java
import org.apache.rocketmq.streams.client.transform.DataStream;
[rocketmq-streams] 07/15: modify the README.md
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 0783741de4caf531e46cbcaea8126fb99fc7ac41
Author: 刈刀 <je...@gmail.com>
AuthorDate: Mon Aug 2 12:16:19 2021 +0800
modify the README.md
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 649319c..5325cdd 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@
## DataStream Example
+
```java
import org.apache.rocketmq.streams.client.transform.DataStream;
[rocketmq-streams] 04/15: add channel-db module
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit ed97ca3daee8bd3e71c4c5150a54381867a07756
Author: vv <ze...@alibaba-inc.com>
AuthorDate: Mon Aug 2 12:03:59 2021 +0800
add channel-db module
---
rocketmq-streams-channel-db/pom.xml | 21 ++
.../streams/db/sink/AbstractMultiTableSink.java | 150 +++++++++++++
.../apache/rocketmq/streams/db/sink/DBSink.java | 239 +++++++++++++++++++++
.../rocketmq/streams/db/sink/DBSinkBuilder.java | 76 +++++++
.../streams/db/sink/SelfMultiTableSink.java | 53 +++++
.../streams/db/sink/SplitBySerialNumber.java | 36 ++++
.../streams/db/sink/SplitByTimeMultiTableSink.java | 36 ++++
.../streams/db/sink/db/DBWriteOnlyChannelTest.java | 84 ++++++++
8 files changed, 695 insertions(+)
diff --git a/rocketmq-streams-channel-db/pom.xml b/rocketmq-streams-channel-db/pom.xml
new file mode 100755
index 0000000..10d760e
--- /dev/null
+++ b/rocketmq-streams-channel-db/pom.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ <name>ROCKETMQ STREAMS :: channel-db</name>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-db-operator</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/AbstractMultiTableSink.java b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/AbstractMultiTableSink.java
new file mode 100644
index 0000000..6547615
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/AbstractMultiTableSink.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink;
+
+import org.apache.rocketmq.streams.common.channel.sinkcache.IMessageFlushCallBack;
+import org.apache.rocketmq.streams.common.channel.sinkcache.impl.MessageCache;
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.functions.MultiTableSplitFunction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+public abstract class AbstractMultiTableSink extends DBSink {
+ protected transient ConcurrentHashMap<String, DBSink> tableSinks = new ConcurrentHashMap();
+ protected transient AtomicLong messageCount = new AtomicLong(0);
+ protected transient MultiTableSplitFunction<IMessage> multiTableSplitFunction;
+
+ public AbstractMultiTableSink(String url, String userName, String password) {
+ this.url = url;
+ this.userName = userName;
+ this.password = password;
+ }
+
+ @Override
+ public boolean batchAdd(IMessage message, ISplit split) {
+
+ DBSink sink = getOrCreateDBSink(split.getQueueId());
+ boolean success = sink.batchAdd(message, split);
+ long count = messageCount.incrementAndGet();
+ if (count >= getBatchSize()) {
+ Set<String> queueIds = new HashSet<>();
+ queueIds.add(split.getQueueId());
+ flush(queueIds);
+ }
+ return success;
+ }
+
+ @Override
+ public boolean batchAdd(IMessage message) {
+ ISplit split = getSplitFromMessage(message);
+ return batchAdd(message, split);
+ }
+
+ @Override
+ public boolean batchSave(List<IMessage> messages) {
+ throw new RuntimeException("can not support this method");
+ }
+
+ @Override
+ public boolean flush(Set<String> splitIds) {
+ if (splitIds == null) {
+ return true;
+ }
+ for (String splitId : splitIds) {
+ DBSink sink = getOrCreateDBSink(splitId);
+ sink.flush();
+ }
+ return true;
+ }
+
+ @Override
+ public boolean flush() {
+ for (DBSink dbSink : tableSinks.values()) {
+ dbSink.flush();
+ }
+ return true;
+ }
+
+ @Override
+ public void openAutoFlush() {
+ for (DBSink dbSink : tableSinks.values()) {
+ dbSink.openAutoFlush();
+ }
+ }
+
+ @Override
+ public void closeAutoFlush() {
+ for (DBSink dbSink : tableSinks.values()) {
+ dbSink.closeAutoFlush();
+ }
+ }
+
+ protected DBSink getOrCreateDBSink(String splitId) {
+ DBSink sink = this.tableSinks.get(splitId);
+ if (sink != null) {
+ return sink;
+ }
+ sink = new DBSink();
+ sink.setUrl(url);
+ sink.setPassword(password);
+ sink.setUserName(userName);
+ sink.setTableName(createTableName(splitId));
+ sink.openAutoFlush();
+ sink.setBatchSize(batchSize);
+ sink.setJdbcDriver(this.jdbcDriver);
+ sink.setMessageCache(new SingleDBSinkCache(sink));
+ sink.init();
+ DBSink existDBSink = this.tableSinks.putIfAbsent(splitId, sink);
+ if (existDBSink != null) {
+ return existDBSink;
+ }
+
+ return sink;
+ }
+
+ protected abstract String createTableName(String splitId);
+
+ protected abstract ISplit getSplitFromMessage(IMessage message);
+
+ protected class SingleDBSinkCache extends MessageCache<IMessage> {
+
+ public SingleDBSinkCache(
+ IMessageFlushCallBack<IMessage> flushCallBack) {
+ super(flushCallBack);
+ }
+
+ @Override
+ public int flush(Set<String> splitIds) {
+ int size = super.flush(splitIds);
+ messageCount.addAndGet(-size);
+ return size;
+ }
+
+ @Override
+ public int flush() {
+ int size = super.flush();
+ messageCount.addAndGet(-size);
+ return size;
+ }
+ }
+
+}
diff --git a/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSink.java b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSink.java
new file mode 100644
index 0000000..5732424
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSink.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.common.channel.IChannel;
+import org.apache.rocketmq.streams.common.channel.sink.AbstractSink;
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.metadata.MetaData;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+
+import java.sql.*;
+import java.util.List;
+
+/**
+ * 主要用于写db，输入可以是一个insert/replace 模版，也可以是metadata对象，二者选一即可。都支持批量插入，提高吞吐 sql 模版：insert into table(column1,column2,column3)values('#{var1}',#{var2},'#{var3}') MetaData:主要是描述每个字段的类型，是否必须 二者选一个即可。sql模式，系统会把一批（batchSize）数据拼成一个大sql。metadata模式，基于字段描述，最终也是拼成一个大sql
+ */
+public class DBSink extends AbstractSink {
+
+ protected String insertSQLTemplate;//完成插入部分的工作，和metadata二选一。insert into table(column1,column2,column3)values('#{var1}',#{var2},'#{var3}')
+
+ protected MetaData metaData;//可以指定meta data，和insertSQL二选一
+
+ protected String tableName; //指定要插入的数据表
+
+ @ENVDependence
+ protected String jdbcDriver = AbstractComponent.DEFAULT_JDBC_DRIVER;
+ @ENVDependence
+ protected String url;
+ @ENVDependence
+ protected String userName;
+ @ENVDependence
+ protected String password;
+
+ /**
+ * db串多数是名字，可以取个名字前缀，如果值为空，默认为此类的name，name为空，默认为简单类名
+ *
+ * @param insertSQL sql模版
+ * @param dbInfoNamePrefix 参数可以是名字，这个是名字前缀.真实值可以配置在配置文件中
+ */
+ public DBSink(String insertSQL, String dbInfoNamePrefix) {
+ setType(IChannel.TYPE);
+ if (StringUtil.isEmpty(dbInfoNamePrefix)) {
+ dbInfoNamePrefix = getConfigureName();
+ }
+ if (StringUtil.isEmpty(dbInfoNamePrefix)) {
+ dbInfoNamePrefix = this.getClass().getSimpleName();
+ }
+ this.insertSQLTemplate = insertSQL;
+ this.url = dbInfoNamePrefix + ".url";
+ this.password = dbInfoNamePrefix + ".password";
+ this.userName = dbInfoNamePrefix + ".userName";
+ }
+
+ public DBSink() {
+ setType(IChannel.TYPE);
+ }
+
+ public DBSink(String url, String userName, String password) {
+ setType(IChannel.TYPE);
+ this.url = url;
+ this.userName = userName;
+ this.password = password;
+ }
+
+ public DBSink(String insertSQL, String url, String userName, String password) {
+ setType(IChannel.TYPE);
+ this.url = url;
+ this.userName = userName;
+ this.password = password;
+ this.insertSQLTemplate = insertSQL;
+ }
+
+ @Override
+ protected boolean initConfigurable() {
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ if (StringUtil.isNotEmpty(this.tableName)) {
+ Connection connection = DriverManager.getConnection(url, userName, password);
+ DatabaseMetaData metaData = connection.getMetaData();
+ ResultSet metaResult = metaData.getColumns(connection.getCatalog(), "%", this.tableName, null);
+ this.metaData = MetaData.createMetaData(metaResult);
+ this.metaData.setTableName(this.tableName);
+ }
+ return super.initConfigurable();
+ } catch (ClassNotFoundException | SQLException e) {
+ e.printStackTrace();
+ }
+ return false;
+ }
+
+ @Override
+ protected boolean batchInsert(List<IMessage> messageList) {
+ JDBCDriver dbDataSource = DriverBuilder.createDriver(jdbcDriver, url, userName, password);
+ try {
+ if (messageList == null || messageList.size() == 0) {
+ return true;
+ }
+ List<JSONObject> messages = convertJsonObjectFromMessage(messageList);
+ if (StringUtil.isEmpty(insertSQLTemplate) && metaData != null) {
+ String sql = SQLUtil.createInsertSql(metaData, messages.get(0));
+ sql = sql + SQLUtil.createInsertValuesSQL(metaData, messages.subList(1, messages.size()));
+ executeSQL(dbDataSource, sql);
+ return true;
+ }
+ String insertValueSQL = parseInsertValues(insertSQLTemplate);
+ if (StringUtil.isEmpty(insertValueSQL) || insertSQLTemplate.replace(insertValueSQL, "").contains("#{")) {
+ for (JSONObject message : messages) {
+ String sql = parseSQL(message, insertSQLTemplate);
+ executeSQL(dbDataSource, sql);
+ }
+ return true;
+ } else {
+ StringBuilder sb = new StringBuilder();
+ String insertSQL;
+ boolean isFirst = true;
+ int i = 0;
+ for (JSONObject message : messages) {
+ insertSQL = parseSQL(message, insertValueSQL);
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ sb.append(",");
+ }
+ i++;
+
+ sb.append(insertSQL);
+ }
+ insertSQL = this.insertSQLTemplate.replace(insertValueSQL, sb.toString());
+ executeSQL(dbDataSource, insertSQL);
+ return true;
+ }
+ } finally {
+ dbDataSource.destroy();
+ }
+ }
+
+ protected void executeSQL(JDBCDriver dbDataSource, String sql) {
+ dbDataSource.execute(sql);
+ }
+
+ /**
+ * 解析出insert value数据部分，对于批量的插入，效果会更佳
+ */
+ private static final String VALUES_NAME = "values";
+
+ protected String parseInsertValues(String insertSQL) {
+ int start = insertSQL.toLowerCase().indexOf(VALUES_NAME);
+ if (start == -1) {
+ return null;
+ }
+ String valuesSQL = insertSQL.substring(start + VALUES_NAME.length());
+ int end = valuesSQL.toLowerCase().lastIndexOf(")");
+ if (end == -1) {
+ return null;
+ }
+ return valuesSQL.substring(0, end + 1);
+ }
+
+ protected String parseSQL(JSONObject message, String sql) {
+ return SQLUtil.parseIbatisSQL(message, sql);
+ }
+
+ public String getInsertSQLTemplate() {
+ return insertSQLTemplate;
+ }
+
+ public void setInsertSQLTemplate(String insertSQLTemplate) {
+ this.insertSQLTemplate = insertSQLTemplate;
+ }
+
+ public String getJdbcDriver() {
+ return jdbcDriver;
+ }
+
+ public void setJdbcDriver(String jdbcDriver) {
+ this.jdbcDriver = jdbcDriver;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public MetaData getMetaData() {
+ return metaData;
+ }
+
+ public void setMetaData(MetaData metaData) {
+ this.metaData = metaData;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+}
diff --git a/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSinkBuilder.java b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSinkBuilder.java
new file mode 100644
index 0000000..ef7ae28
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/DBSinkBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.channel.builder.IChannelBuilder;
+import org.apache.rocketmq.streams.common.channel.sink.ISink;
+import org.apache.rocketmq.streams.common.channel.source.ISource;
+import org.apache.rocketmq.streams.common.model.ServiceName;
+import org.apache.rocketmq.streams.common.metadata.MetaData;
+import org.apache.rocketmq.streams.common.metadata.MetaDataField;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import com.google.auto.service.AutoService;
+
+@AutoService(IChannelBuilder.class)
+@ServiceName(DBSinkBuilder.TYPE)
+public class DBSinkBuilder implements IChannelBuilder {
+ public static final String TYPE = "rds";
+
+ @Override
+ public ISink createSink(String namespace, String name, Properties properties, MetaData metaData) {
+ DBSink sink = new DBSink();
+ sink.setUrl(properties.getProperty("url"));
+ sink.setUserName("userName");
+ sink.setPassword("password");
+ List<MetaDataField> fieldList = metaData.getMetaDataFields();
+ StringBuilder insertSQL = new StringBuilder();
+ StringBuilder insertValueSQL = new StringBuilder();
+ boolean isFirst = true;
+ for (MetaDataField field : fieldList) {
+ String fieldName = field.getFieldName();
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ insertSQL.append(",");
+ insertValueSQL.append(",");
+ }
+ insertSQL.append(fieldName);
+ if (DataTypeUtil.isNumber(field.getDataType())) {
+ insertValueSQL.append(fieldName);
+ } else {
+ insertValueSQL.append("'#{" + fieldName + "}'");
+ }
+ }
+ String sql = "insert into " + properties.getProperty("tableName") + "(" + insertSQL.toString() + ")values(" + insertValueSQL.toString() + ")";
+ sink.setInsertSQLTemplate(sql);
+ return sink;
+ }
+
+ @Override
+ public ISource createSource(String namespace, String name, Properties properties, MetaData metaData) {
+ throw new RuntimeException("can not support this method");
+ }
+
+ @Override
+ public String getType() {
+ return TYPE;
+ }
+
+}
diff --git a/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SelfMultiTableSink.java b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SelfMultiTableSink.java
new file mode 100644
index 0000000..96922e6
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SelfMultiTableSink.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink;
+
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.functions.MultiTableSplitFunction;
+import org.apache.rocketmq.streams.common.utils.Base64Utils;
+import org.apache.rocketmq.streams.common.utils.InstantiationUtil;
+
+public class SelfMultiTableSink extends AbstractMultiTableSink implements IAfterConfiguableRefreshListerner {
+ protected String multiTableSplitFunctionSerializeValue;//用户自定义的operator的序列化字节数组，做了base64解码
+ protected transient MultiTableSplitFunction<IMessage> multiTableSplitFunction;
+
+ public SelfMultiTableSink(String url, String userName, String password, MultiTableSplitFunction<IMessage> multiTableSplitFunction) {
+ super(url, userName, password);
+ this.multiTableSplitFunction = multiTableSplitFunction;
+ byte[] bytes = InstantiationUtil.serializeObject(multiTableSplitFunction);
+ multiTableSplitFunctionSerializeValue = Base64Utils.encode(bytes);
+ }
+
+ @Override
+ protected String createTableName(String splitId) {
+ return multiTableSplitFunction.createTableFromSplitId(splitId);
+ }
+
+ @Override
+ protected ISplit getSplitFromMessage(IMessage message) {
+ return multiTableSplitFunction.createSplit(message);
+ }
+
+ @Override
+ public void doProcessAfterRefreshConfigurable(IConfigurableService configurableService) {
+ byte[] bytes = Base64Utils.decode(multiTableSplitFunctionSerializeValue);
+ this.multiTableSplitFunction = InstantiationUtil.deserializeObject(bytes);
+ }
+}
diff --git a/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitBySerialNumber.java b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitBySerialNumber.java
new file mode 100644
index 0000000..c2a49b7
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitBySerialNumber.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink;
+
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.context.IMessage;
+
+public class SplitBySerialNumber extends AbstractMultiTableSink {
+ public SplitBySerialNumber(String url, String userName, String password) {
+ super(url, userName, password);
+ }
+
+ @Override
+ protected String createTableName(String splitId) {
+ return null;
+ }
+
+ @Override
+ protected ISplit getSplitFromMessage(IMessage message) {
+ return null;
+ }
+}
diff --git a/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitByTimeMultiTableSink.java b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitByTimeMultiTableSink.java
new file mode 100644
index 0000000..87a2b3e
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/main/java/org/apache/rocketmq/streams/db/sink/SplitByTimeMultiTableSink.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink;
+
+import org.apache.rocketmq.streams.common.channel.split.ISplit;
+import org.apache.rocketmq.streams.common.context.IMessage;
+
+public class SplitByTimeMultiTableSink extends AbstractMultiTableSink {
+ public SplitByTimeMultiTableSink(String url, String userName, String password) {
+ super(url, userName, password);
+ }
+
+ @Override
+ protected String createTableName(String splitId) {
+ return null;
+ }
+
+ @Override
+ protected ISplit getSplitFromMessage(IMessage message) {
+ return null;
+ }
+}
diff --git a/rocketmq-streams-channel-db/src/test/java/org/apache/rocketmq/streams/db/sink/db/DBWriteOnlyChannelTest.java b/rocketmq-streams-channel-db/src/test/java/org/apache/rocketmq/streams/db/sink/db/DBWriteOnlyChannelTest.java
new file mode 100644
index 0000000..c14bbc3
--- /dev/null
+++ b/rocketmq-streams-channel-db/src/test/java/org/apache/rocketmq/streams/db/sink/db/DBWriteOnlyChannelTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.db.sink.db;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.context.Message;
+import org.apache.rocketmq.streams.common.metadata.MetaData;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.sink.DBSink;
+import org.junit.Test;
+
+public class DBWriteOnlyChannelTest {
+
+ private String URL = "jdbc:mysql://XXXXX:3306/yundun_soc?useUnicode=true&characterEncoding=utf8&autoReconnect=true";
+ protected String USER_NAME = "XXXX";
+ protected String PASSWORD = "XXXX";
+
+ @Test
+ public void testOutputBySQL() {
+ String sql = "insert into table(name,age) values('#{name}',#{age})";
+ DBSink sink = new DBSink(sql, URL, USER_NAME, PASSWORD) {
+
+ /**
+ * 因为不是真实表，会报错，把执行sql，改成打印sql
+ */
+ @Override
+ protected void executeSQL(JDBCDriver dbDataSource, String sql) {
+ System.out.println(sql);
+ }
+ };
+ for (int i = 0; i < 10; i++) {
+ JSONObject msg = new JSONObject();
+ msg.put("name", "chris" + i);
+ msg.put("age", i);
+ IMessage message = new Message(msg);
+ sink.batchAdd(message);
+ }
+ sink.flush();
+ }
+
+ @Test
+ public void testOutputByMetaData() {
+ DBSink sink = new DBSink() {
+ /**
+ * 因为不是真实表，会报错，把执行sql，改成打印sql
+ */
+ @Override
+ protected void executeSQL(JDBCDriver dbDataSource, String sql) {
+ System.out.println(sql);
+ }
+ };
+ JSONObject msg = new JSONObject();
+ msg.put("name", "chris");
+ msg.put("age", 18);
+ MetaData metaData = MetaData.createMetaData(msg);
+ metaData.setTableName("tableName");
+ sink.setMetaData(metaData);
+ for (int i = 0; i < 10; i++) {
+ msg = new JSONObject();
+ msg.put("name", "chris" + i);
+ msg.put("age", i);
+ IMessage message = new Message(msg);
+ sink.batchAdd(message);
+ }
+ sink.flush();
+ }
+
+}
[rocketmq-streams] 01/15: first commit
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit 997f3dfc3b8ba4d3feaca97298e8da741be44d24
Author: duhenglucky <du...@apache.org>
AuthorDate: Tue Jul 13 21:07:47 2021 +0800
first commit
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..49cd84d
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# rocketmq-streams