You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by du...@apache.org on 2021/08/03 06:50:30 UTC
[rocketmq-streams] 02/35: add lease、dim and client module
This is an automated email from the ASF dual-hosted git repository.
duhengforever pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit b757f8db8141d282dd793d37af9b509bcc54b7ca
Author: 刈刀 <ju...@alibaba-inc.com>
AuthorDate: Mon Aug 2 11:21:45 2021 +0800
add lease、dim and client module
---
.gitignore | 25 ++
README.md | 108 +++++
pom.xml | 410 +++++++++++++++++++
rocketmq-streams-clients/pom.xml | 46 +++
.../rocketmq/streams/client/DataStreamAction.java | 101 +++++
.../rocketmq/streams/client/StreamBuilder.java | 28 ++
.../streams/client/source/DataStreamSource.java | 76 ++++
.../client/strategy/CheckpointStrategy.java | 69 ++++
.../streams/client/strategy/StateStrategy.java | 37 ++
.../rocketmq/streams/client/strategy/Strategy.java | 25 ++
.../streams/client/transform/DataStream.java | 437 +++++++++++++++++++++
.../streams/client/transform/JoinStream.java | 212 ++++++++++
.../streams/client/transform/SplitStream.java | 61 +++
.../streams/client/transform/WindowStream.java | 210 ++++++++++
.../client/transform/window/HoppingWindow.java | 32 ++
.../client/transform/window/SessionWindow.java | 32 ++
.../streams/client/transform/window/Time.java | 45 +++
.../client/transform/window/TumblingWindow.java | 33 ++
.../client/transform/window/WindowInfo.java | 83 ++++
.../rocketmq/streams/client/DBDriverTest.java | 75 ++++
.../rocketmq/streams/client/DataStreamTest.java | 107 +++++
.../apache/rocketmq/streams/client/FilterTest.java | 49 +++
.../apache/rocketmq/streams/client/JoinTest.java | 89 +++++
.../apache/rocketmq/streams/client/LeaseTest.java | 98 +++++
.../rocketmq/streams/client/ORMUtilTest.java | 172 ++++++++
.../apache/rocketmq/streams/client/SplitTest.java | 86 ++++
.../apache/rocketmq/streams/client/UnionTest.java | 82 ++++
.../apache/rocketmq/streams/client/WindowTest.java | 86 ++++
.../client/windows/AbstractWindowFireModeTest.java | 189 +++++++++
.../streams/client/windows/WindowFromFileTest.java | 158 ++++++++
.../streams/client/windows/WindowFromMetaq.java | 47 +++
.../client/windows/WindowHighAvailabilityTest.java | 131 ++++++
.../src/test/resources/log4j.xml | 36 ++
rocketmq-streams-dim/pom.xml | 47 +++
.../apache/rocketmq/streams/dim/DimComponent.java | 63 +++
.../rocketmq/streams/dim/builder/DimBuilder.java | 94 +++++
.../function/expression/InExpressionResource.java | 80 ++++
.../expression/NotInExpressionResource.java | 45 +++
.../dim/function/script/IntelligenceFunction.java | 81 ++++
.../script/IntelligenceNameListFunction.java | 24 ++
.../dim/function/script/NameListFunction.java | 203 ++++++++++
.../rocketmq/streams/dim/index/DimIndex.java | 319 +++++++++++++++
.../rocketmq/streams/dim/index/IndexExecutor.java | 258 ++++++++++++
.../intelligence/AbstractIntelligenceCache.java | 395 +++++++++++++++++++
.../dim/intelligence/AccountIntelligenceCache.java | 77 ++++
.../dim/intelligence/DomainIntelligenceCache.java | 83 ++++
.../dim/intelligence/IPIntelligenceCache.java | 108 +++++
.../dim/intelligence/URLIntelligenceCache.java | 80 ++++
.../rocketmq/streams/dim/model/AbstractDim.java | 312 +++++++++++++++
.../streams/dim/model/BooleanFieldDBDim.java | 55 +++
.../apache/rocketmq/streams/dim/model/DBDim.java | 140 +++++++
.../rocketmq/streams/dim/service/IDimService.java | 65 +++
.../streams/dim/service/impl/DimServiceImpl.java | 92 +++++
.../com/aliyun/service/ConfigureLoaderTest.java | 37 ++
.../com/aliyun/service/ExpressionExecutorTest.java | 80 ++++
.../java/com/aliyun/service/JsonParserTest.java | 40 ++
.../com/aliyun/service/NameListFunctionTest.java | 90 +++++
.../java/com/aliyun/service/TableCompressTest.java | 26 ++
rocketmq-streams-lease/pom.xml | 25 ++
.../rocketmq/streams/lease/LeaseComponent.java | 103 +++++
.../rocketmq/streams/lease/model/LeaseInfo.java | 127 ++++++
.../streams/lease/service/ILeaseGetCallback.java | 30 ++
.../streams/lease/service/ILeaseService.java | 136 +++++++
.../streams/lease/service/ILeaseStorage.java | 73 ++++
.../streams/lease/service/ILeaseStorasge.java | 63 +++
.../lease/service/impl/BasedLesaseImpl.java | 404 +++++++++++++++++++
.../lease/service/impl/LeaseServiceImpl.java | 275 +++++++++++++
.../streams/lease/service/impl/MockLeaseImpl.java | 95 +++++
.../lease/service/storages/DBLeaseStorage.java | 229 +++++++++++
.../rocketmq/streams/lease/LeaseComponentTest.java | 119 ++++++
.../src/test/resources/log4j.xml | 20 +
71 files changed, 8068 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1471653
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,25 @@
+target/
+.DS_Store
+!.mvn/wrapper/maven-wrapper.jar
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+nbproject/private/
+build/
+nbbuild/
+dist/
+nbdist/
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5f3cde5
--- /dev/null
+++ b/README.md
@@ -0,0 +1,108 @@
+# Rocketmq Streams
+
+## Features
+
+* 轻量级部署：可以单独部署，也支持集群部署
+* 多种类型的数据输入以及输出，source支持 rocketmq ， sink支持db, rocketmq 等
+
+## DataStream Example
+
+```java
+import org.apache.rocketmq.streams.client.transform.DataStream;
+
+DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+
+ source
+ .fromFile("/Users/junjie.cheng/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
+```
+
+## Maven Repository
+
+```xml
+
+<dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+</dependency>
+```
+
+# Core API
+
+rocketmq-stream 实现了一系列高级的API，可以让用户很方便的编写流计算的程序，实现自己的业务需求；
+
+## StreamBuilder
+
+StreamBuilder 用于构建流任务的源； 内部包含```dataStream()```和```tableStream()```俩个方法，分别返回DataStreamSource和TableStreamSource俩个源；
+
++ [dataStream(nameSpaceName,pipelineName)]() 返回DataStreamSource实例，用于分段编程实现流计算任务；
+
+## DataStream API
+
+### Source
+
+DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据；
+
++ ```fromFile``` 从文件中读取数据， 该方法包含俩个参数
+ + ```filePath``` 文件路径，必填参数
+ + ```isJsonData``` 是否json数据， 非必填参数， 默认为```true```
+
+
++ ```fromRocketmq``` 从rocketmq中获取数据，包含四个参数
+ + ```topic``` rocketmq消息队列的topic名称，必填参数
+ + ```groupName``` 消费者组的名称，必填参数
+ + ```isJson``` 是否json格式，非必填参数
+ + ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
+
+
++ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
+
+### transform
+
+transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
+
+#### DataStream
+
+DataStream实现了一系列常见的流计算算子
+
++ ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream
++ ```flatmap``` 与map类似，一个输入项对应0个或者多个输出项
++ ```filter``` 只选择func返回true的源DStream的记录来返回一个新的DStream
++ ```forEach``` 对每个记录执行一次函数func， 返回一个新的DataStream
++ ```selectFields``` 对每个记录返回对应的字段值，返回一个新的DataStream
++ ```operate``` 对每个记录执行一次自定义的函数，返回一个新的DataStream
++ ```script``` 针对每个记录的字段执行一段脚本，返回新的字段，生成一个新的DataStream
++ ```toPrint``` 将结果在控制台打印，生成新的DataStreamAction实例
++ ```toFile``` 将结果保存为文件，生成一个新的DataStreamAction实例
++ ```toDB``` 将结果保存到数据库
++ ```toRocketmq``` 将结果输出到rocketmq
++ ```toSls``` 将结果输出到sls
++ ```to``` 将结果经过自定义的ISink接口输出到指定的存储
++ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
+ + ```count``` 在窗口内计数
+ + ```min``` 获取窗口内统计值的最小值
+ + ```max``` 获取窗口内统计值得最大值
+ + ```avg``` 获取窗口内统计值的平均值
+ + ```sum``` 获取窗口内统计值的加和值
+ + ```reduce``` 在窗口内进行自定义的汇总运算
++ ```join``` 根据条件将将俩个流进行关联， 合并为一个大流进行相关的运算
++ ```union``` 将俩个流进行合并
++ ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算
++ ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等
+
+# Strategy
+
+策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
+
+```java
+//指定checkpoint的存储策略
+source
+ .fromRocketmq("TSG_META_INFO","")
+ .map(message->message+"--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
+ .start();
+```
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..3675792
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,410 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <name>ROCKETMQ STREAMS</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>rocketmq-streams-commons</module>
+ <module>rocketmq-streams-dim</module>
+ <module>rocketmq-streams-transport-minio</module>
+ <module>rocketmq-streams-script</module>
+ <module>rocketmq-streams-script-python</module>
+ <module>rocketmq-streams-configurable</module>
+ <module>rocketmq-streams-serviceloader</module>
+ <module>rocketmq-streams-filter</module>
+ <module>rocketmq-streams-schedule</module>
+ <module>rocketmq-streams-lease</module>
+ <module>rocketmq-streams-db-operator</module>
+ <module>rocketmq-streams-window</module>
+ <module>rocketmq-streams-clients</module>
+ <module>rocketmq-streams-channel-rocketmq</module>
+ <module>rocketmq-streams-channel-db</module>
+ <module>rocketmq-streams-channel-http</module>
+ </modules>
+
+ <properties>
+ <java.version>1.8</java.version>
+ <java.encoding>UTF-8</java.encoding>
+ <project.build.sourceEncoding>${java.encoding}</project.build.sourceEncoding>
+ <log4j.version>1.2.17</log4j.version>
+ <commons-logging.version>1.1</commons-logging.version>
+ <spring.version>3.2.13.RELEASE</spring.version>
+ <auto-service.version>1.0-rc5</auto-service.version>
+ <mysql-connector.version>5.1.40</mysql-connector.version>
+ <fastjson.version>1.2.27</fastjson.version>
+ <quartz.version>2.2.1</quartz.version>
+ <httpclient.version>4.5.2</httpclient.version>
+ <commons-io.version>2.5</commons-io.version>
+ <junit.version>4.12</junit.version>
+ <guava.version>25.1-jre</guava.version>
+ <groovy.version>2.1.8</groovy.version>
+ <disruptor.version>3.2.0</disruptor.version>
+ <rocksdbjni.version>6.6.4</rocksdbjni.version>
+ <rocketmq.version>4.5.2</rocketmq.version>
+ <hyperscan.version>5.4.0-2.0.0</hyperscan.version>
+ <platform.version>3.5.2</platform.version>
+ <gson.version>2.8.5</gson.version>
+ <java-grok.version>0.1.9</java-grok.version>
+ <jython.version>2.7.0</jython.version>
+ <scala-library.version>2.12.4</scala-library.version>
+ <logback-core.version>1.2.2</logback-core.version>
+ <minio.version>3.0.10</minio.version>
+ </properties>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.18.1</version>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.5.1</version>
+ <configuration>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
+ <encoding>UTF-8</encoding>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>versions-maven-plugin</artifactId>
+ <version>2.2</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- ================================================= -->
+ <!-- rocketmq streams library -->
+ <!-- ================================================= -->
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-configurable</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-db-operator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-dim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-filter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-lease</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-schedule</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-script</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-script-python</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-serviceloader</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-transport-minio</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-window</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-rocketmq</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- ================================================= -->
+ <!-- rocketmq library -->
+ <!-- ================================================= -->
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-tools</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-common</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-acl</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+
+ <!-- ================================================= -->
+ <!-- tool library -->
+ <!-- ================================================= -->
+
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>${fastjson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons-logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <version>${auto-service.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>${disruptor.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.gliwka.hyperscan</groupId>
+ <artifactId>hyperscan</artifactId>
+ <version>${hyperscan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>platform</artifactId>
+ <version>${platform.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql-connector.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz-jobs</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.krakens</groupId>
+ <artifactId>java-grok</artifactId>
+ <version>${java-grok.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <version>${groovy.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython-standalone</artifactId>
+ <version>${jython.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala-library.version}</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>${logback-core.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio</artifactId>
+ <version>${minio.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ <version>${rocksdbjni.version}</version>
+ </dependency>
+
+ </dependencies>
+ </dependencyManagement>
+
+</project>
diff --git a/rocketmq-streams-clients/pom.xml b/rocketmq-streams-clients/pom.xml
new file mode 100644
index 0000000..ffc051a
--- /dev/null
+++ b/rocketmq-streams-clients/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>rocketmq-streams</artifactId>
+ <groupId>org.apache.rocketmq</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>rocketmq-streams-clients</artifactId>
+ <name>ROCKETMQ STREAMS :: clients</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-commons</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-rocketmq</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-script</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-filter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-window</artifactId>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+
+</project>
\ No newline at end of file
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/DataStreamAction.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/DataStreamAction.java
new file mode 100644
index 0000000..105bd8e
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/DataStreamAction.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.google.common.collect.Maps;
+import org.apache.rocketmq.streams.client.strategy.Strategy;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.topology.ChainPipeline;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+public class DataStreamAction extends DataStream {
+
+ private final Map<String, Object> properties = Maps.newHashMap();
+
+ public DataStreamAction(String namespace, String pipelineName) {
+ super(namespace, pipelineName);
+ }
+
+ public DataStreamAction(PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ super(pipelineBuilder, pipelineBuilders, currentChainStage);
+ }
+
+ public DataStreamAction with(Strategy... strategies) {
+ Properties properties = new Properties();
+ for (Strategy strategy : strategies) {
+ properties.putAll(strategy.getStrategyProperties());
+ }
+ ComponentCreator.createProperties(properties);
+ return this;
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void start() {
+ start(false);
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void asyncStart() {
+ start(true);
+ }
+
+ protected void start(boolean isAsync) {
+ if (this.mainPipelineBuilder == null) {
+ return;
+ }
+ properties.put(ConfigureFileKey.CONNECT_TYPE, "memory");
+ String[] kvs = new String[properties.size()];
+ int i = 0;
+ for (Map.Entry<String, Object> entry : properties.entrySet()) {
+ kvs[i++] = entry.getKey() + ":" + entry.getValue();
+ }
+
+ ConfigurableComponent configurableComponent = ComponentCreator.getComponent(mainPipelineBuilder.getPipelineNameSpace(), ConfigurableComponent.class, kvs);
+ ChainPipeline pipeline = this.mainPipelineBuilder.build(configurableComponent.getService());
+ pipeline.startChannel();
+ if (this.otherPipelineBuilders != null) {
+ for (PipelineBuilder builder : otherPipelineBuilders) {
+ ChainPipeline otherPipeline = builder.build(configurableComponent.getService());
+ otherPipeline.startChannel();
+ }
+ }
+ if (isAsync) {
+ return;
+ }
+ while (true) {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java
new file mode 100644
index 0000000..f67ee2c
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/StreamBuilder.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import org.apache.rocketmq.streams.client.source.DataStreamSource;
+
+public class StreamBuilder {
+
+ public static DataStreamSource dataStream(String nameSpaceName, String pipelineName) {
+ return DataStreamSource.create(nameSpaceName, pipelineName);
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
new file mode 100644
index 0000000..8d71c31
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/source/DataStreamSource.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.source;
+
+import com.google.common.collect.Sets;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.channel.impl.file.FileSource;
+import org.apache.rocketmq.streams.common.channel.source.ISource;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.source.RocketMQSource;
+
+import java.util.Set;
+
+public class DataStreamSource {
+ protected PipelineBuilder mainPipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+
+ public DataStreamSource(String namespace, String pipelineName) {
+ this.mainPipelineBuilder = new PipelineBuilder(namespace, pipelineName);
+ this.otherPipelineBuilders = Sets.newHashSet();
+ }
+
+ public static DataStreamSource create(String namespace, String pipelineName) {
+ return new DataStreamSource(namespace, pipelineName);
+ }
+
+ public DataStream fromFile(String filePath) {
+ return fromFile(filePath, true);
+ }
+
+ public DataStream fromFile(String filePath, Boolean isJsonData) {
+ FileSource fileChannel = new FileSource(filePath);
+ fileChannel.setJsonData(isJsonData);
+ this.mainPipelineBuilder.setSource(fileChannel);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
+ }
+
+ public DataStream fromRocketmq(String topic, String groupName) {
+ return fromRocketmq(topic, groupName, null, false);
+ }
+
+ public DataStream fromRocketmq(String topic, String groupName, boolean isJson) {
+ return fromRocketmq(topic, groupName, null, isJson);
+ }
+
+ public DataStream fromRocketmq(String topic, String groupName, String tags, boolean isJson) {
+ RocketMQSource rocketMQSource = new RocketMQSource();
+ rocketMQSource.setTopic(topic);
+ rocketMQSource.setTags(tags);
+ rocketMQSource.setGroupName(groupName);
+ rocketMQSource.setJsonData(isJson);
+ this.mainPipelineBuilder.setSource(rocketMQSource);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
+ }
+
+ public DataStream from(ISource<?> source) {
+ this.mainPipelineBuilder.setSource(source);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, null);
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/CheckpointStrategy.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/CheckpointStrategy.java
new file mode 100644
index 0000000..20494c4
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/CheckpointStrategy.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.strategy;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+
+import java.util.Properties;
+
+public class CheckpointStrategy implements Strategy {
+
+ private final Properties properties;
+
+ private CheckpointStrategy(Long pollingTime) {
+ properties = new Properties();
+ properties.put(AbstractComponent.CONNECT_TYPE, IConfigurableService.MEMORY_SERVICE_NAME);
+ properties.put(AbstractComponent.POLLING_TIME, pollingTime + "");
+ }
+
+ private CheckpointStrategy(String filePath, Long pollingTime) {
+ properties = new Properties();
+ properties.put(AbstractComponent.CONNECT_TYPE, IConfigurableService.FILE_SERVICE_NAME);
+ properties.put(IConfigurableService.FILE_PATH_NAME, filePath);
+ properties.put(AbstractComponent.POLLING_TIME, pollingTime + "");
+ }
+
+ private CheckpointStrategy(String url, String username, String password, Long pollingTime) {
+ properties = new Properties();
+ properties.put(AbstractComponent.JDBC_DRIVER, AbstractComponent.DEFAULT_JDBC_DRIVER);
+ properties.put(AbstractComponent.JDBC_URL, url);
+ properties.put(AbstractComponent.JDBC_USERNAME, username);
+ properties.put(AbstractComponent.JDBC_PASSWORD, password);
+ properties.put(AbstractComponent.JDBC_TABLE_NAME, AbstractComponent.DEFAULT_JDBC_TABLE_NAME);
+ properties.put(AbstractComponent.POLLING_TIME, pollingTime + "");
+ properties.put(AbstractComponent.CONNECT_TYPE, IConfigurableService.DEFAULT_SERVICE_NAME);
+ }
+
+ @Override
+ public Properties getStrategyProperties() {
+ return this.properties;
+ }
+
+ public static Strategy db(String url, String username, String password, Long pollingTime) {
+ return new CheckpointStrategy(url, username, password, pollingTime);
+ }
+
+ public static Strategy file(String filePath, Long pollingTime) {
+ return new CheckpointStrategy(filePath, pollingTime);
+ }
+
+ public static Strategy mem(Long pollingTime) {
+ return new CheckpointStrategy(pollingTime);
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java
new file mode 100644
index 0000000..c647759
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/StateStrategy.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.strategy;
+
+import java.util.Properties;
+
+public class StateStrategy implements Strategy {
+
+ private Properties properties;
+
+ private StateStrategy() {
+ }
+
+ @Override
+ public Properties getStrategyProperties() {
+ return this.properties;
+ }
+
+ public static Strategy db() {
+ return new StateStrategy();
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/Strategy.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/Strategy.java
new file mode 100644
index 0000000..727c645
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/strategy/Strategy.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.strategy;
+
+import java.util.Properties;
+
+public interface Strategy {
+
+ Properties getStrategyProperties();
+
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
new file mode 100644
index 0000000..41aae96
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/DataStream.java
@@ -0,0 +1,437 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Sets;
+import org.apache.rocketmq.streams.client.DataStreamAction;
+import org.apache.rocketmq.streams.client.transform.window.WindowInfo;
+import org.apache.rocketmq.streams.common.channel.impl.OutputPrintChannel;
+import org.apache.rocketmq.streams.common.channel.impl.file.FileSink;
+import org.apache.rocketmq.streams.common.channel.sink.ISink;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.context.AbstractContext;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.context.MessageHeader;
+import org.apache.rocketmq.streams.common.context.UserDefinedMessage;
+import org.apache.rocketmq.streams.common.functions.*;
+import org.apache.rocketmq.streams.common.topology.ChainPipeline;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.IStageBuilder;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.topology.model.Union;
+import org.apache.rocketmq.streams.common.topology.stages.udf.StageBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.udf.UDFUnionChainStage;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.db.sink.DBSink;
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.apache.rocketmq.streams.filter.operator.FilterOperator;
+import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator;
+import org.apache.rocketmq.streams.sink.RocketMQSink;
+import org.apache.rocketmq.streams.window.builder.WindowBuilder;
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+import org.apache.rocketmq.streams.window.operator.join.JoinWindow;
+
+import java.io.Serializable;
+import java.util.Set;
+
+public class DataStream implements Serializable {
+
+ protected PipelineBuilder mainPipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+ public DataStream(String namespace, String pipelineName) {
+ this.mainPipelineBuilder = new PipelineBuilder(namespace, pipelineName);
+ this.otherPipelineBuilders = Sets.newHashSet();
+ }
+
+ public DataStream(PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.mainPipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ }
+
+ public DataStream script(String script) {
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(new ScriptOperator(script));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ public DataStream filter(String expressions) {
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(new FilterOperator(expressions));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ public <T, O> DataStream map(MapFunction<T, O> mapFunction) {
+ StageBuilder stageBuilder = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ try {
+ O o = (O)(message.getMessageValue());
+ T result = (T)mapFunction.map(o);
+ if (result != message.getMessageValue()) {
+ if (result instanceof JSONObject) {
+ message.setMessageBody((JSONObject)result);
+ } else {
+ message.setMessageBody(new UserDefinedMessage(result));
+ }
+ }
+ return null;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ };
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(stageBuilder);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ public <O> DataStream filter(final FilterFunction<O> filterFunction) {
+ StageBuilder mapUDFOperator = new StageBuilder() {
+
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ try {
+ boolean isFilter = filterFunction.filter((O)message.getMessageValue());
+ if (isFilter) {
+ context.breakExecute();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ };
+ ChainStage stage = this.mainPipelineBuilder.createStage(mapUDFOperator);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * windows streams
+ *
+ * @param windowInfo 通过不同窗口类型动of方法创建，SessionWindow.of(Time.seconds(10))
+ * @return WindowStream
+ */
+ public WindowStream window(WindowInfo windowInfo) {
+ AbstractWindow window = windowInfo.createWindow();
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(window);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new WindowStream(window, this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 通用增加stage的方法，低级接口，适合用户自定义stage的场景
+ *
+ * @param stageBuilder 创建stage和变量的接口
+ * @return DataStream
+ */
+ public DataStream addStage(IStageBuilder stageBuilder) {
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(stageBuilder);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 创建join stream。实现原理，通过共享一个JoinWindow，并打标左右流，在join windown完成缓存，join逻辑
+ *
+ * @param rightStream 通过不同窗口类型动of方法创建，SessionWindow.of(Time.seconds(10))
+ * @return
+ */
+ public JoinStream join(DataStream rightStream) {
+ JoinWindow window = WindowBuilder.createDefaultJoinWindow();
+ //处理左边分支，增加map，主要是增加msg msgRouteFromLable->增加窗口stage
+ ChainStage<?> leftScriptStage = this.mainPipelineBuilder.createStage(new ScriptOperator("setHeader(msgRouteFromLable,'" + MessageHeader.JOIN_LEFT + "')"));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, leftScriptStage);
+ this.currentChainStage = leftScriptStage;
+ ChainStage<?> leftWindowStage = this.mainPipelineBuilder.createStage(window);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, leftWindowStage);
+
+ //处理右流，右流增加script
+ DataStream dataStream = rightStream.script("setHeader(msgRouteFromLable,'" + MessageHeader.JOIN_RIGHT + "')").addStage(window);
+ //dataStream.addStage(window);
+
+ addOtherDataStream(rightStream);
+ return new JoinStream(window, this.mainPipelineBuilder, this.otherPipelineBuilders, leftWindowStage);
+ }
+
+ /**
+ * 通过共享对象union，完成两个数据汇聚，左流需要设置isMainStream=true
+ *
+ * @param rightStream
+ * @return
+ */
+ public DataStream union(DataStream rightStream) {
+ Union union = new Union();
+
+ //处理左流，做流的isMain设置成true
+ UDFUnionChainStage chainStage = (UDFUnionChainStage)this.mainPipelineBuilder.createStage(union);
+ chainStage.setMainStream(true);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, chainStage);
+
+ //处理右流，做流的isMain设置成true
+ rightStream.addStage(union);
+
+ addOtherDataStream(rightStream);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, chainStage);
+ }
+
+ /**
+ * 把一个流拆分成多个流，通过设置不同流的标签实现
+ *
+ * @param splitFunction 拆分流的具体逻辑
+ * @return
+ */
+ public SplitStream split(SplitFunction splitFunction) {
+ StageBuilder operator = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ String labelName = splitFunction.split(message.getMessageValue());
+ message.getHeader().addRouteLable(labelName);
+ return null;
+ }
+ };
+ ChainStage<?> stage = this.mainPipelineBuilder.createStage(operator);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new SplitStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 维表join,mysql场景，不需要指定jdbcdriver
+ *
+ * @param url
+ * @param userName
+ * @param password
+ * @param sqlOrTableName
+ * @return
+ */
+ public JoinStream join(String url, String userName, String password, String sqlOrTableName, long pollingTimeMintue) {
+ return join(url, userName, password, sqlOrTableName, null, pollingTimeMintue);
+ }
+
+ /**
+ * 维表join
+ *
+ * @param url
+ * @param userName
+ * @param password
+ * @param sqlOrTableName
+ * @return
+ */
+ public JoinStream join(String url, String userName, String password, String sqlOrTableName, String jdbcDriver, long pollingTimeMinute) {
+ DBDim dbDim = new DBDim();
+ dbDim.setUrl(url);
+ dbDim.setUserName(userName);
+ dbDim.setPassword(password);
+ dbDim.setSql(sqlOrTableName);
+ dbDim.setPollingTimeMintue(pollingTimeMinute);
+ dbDim.setJdbcdriver(jdbcDriver);
+ this.mainPipelineBuilder.addConfigurables(dbDim);
+ return new JoinStream(dbDim, mainPipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+
+ /**
+ * 遍历所有数据
+ *
+ * @param forEachFunction
+ * @param <O>
+ * @return
+ */
+ public <O> DataStream forEach(ForEachFunction<O> forEachFunction) {
+ StageBuilder selfChainStage = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ forEachFunction.foreach((O)message.getMessageValue());
+ return null;
+ }
+ };
+ ChainStage stage = this.mainPipelineBuilder.createStage(selfChainStage);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 遍历所有数据
+ *
+ * @param forEachFunction
+ * @param <O>
+ * @return
+ */
+ public <O> DataStream forEachMessage(ForEachMessageFunction forEachFunction) {
+ StageBuilder selfChainStage = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ forEachFunction.foreach(message, context);
+ return null;
+ }
+ };
+ ChainStage stage = this.mainPipelineBuilder.createStage(selfChainStage);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 只保留需要的字段
+ *
+ * @param fieldNames
+ */
+ public DataStream selectFields(String... fieldNames) {
+ ChainStage stage = this.mainPipelineBuilder.createStage(new ScriptOperator("retain(" + MapKeyUtil.createKeyBySign(",", fieldNames) + ")"));
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(this.mainPipelineBuilder, this.otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void start() {
+ start(false);
+ }
+
+ /**
+ * 启动流任务
+ */
+ public void asynStart() {
+ start(true);
+ }
+
+ protected void start(boolean isAsyn) {
+ if (this.mainPipelineBuilder == null) {
+ return;
+ }
+ ConfigurableComponent configurableComponent = ComponentCreator.getComponent(mainPipelineBuilder.getPipelineNameSpace(), ConfigurableComponent.class, ConfigureFileKey.CONNECT_TYPE + ":memory");
+ ChainPipeline pipeline = this.mainPipelineBuilder.build(configurableComponent.getService());
+ pipeline.startChannel();
+ if (this.otherPipelineBuilders != null) {
+ for (PipelineBuilder builder : otherPipelineBuilders) {
+ ChainPipeline otherPipeline = builder.build(configurableComponent.getService());
+ otherPipeline.startChannel();
+ }
+ }
+ if (isAsyn) {
+ return;
+ }
+ while (true) {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * 把其他流的 pipelinebuilder 放到set中
+ *
+ * @param rightSource
+ */
+ protected void addOtherDataStream(DataStream rightSource) {
+ //如果是多流join，需要把把piplinebuider保存下来，在启动时，启动多个pipline
+ if (!rightSource.mainPipelineBuilder.equals(this.mainPipelineBuilder)) {
+ this.otherPipelineBuilders.add(rightSource.mainPipelineBuilder);
+ }
+
+ this.otherPipelineBuilders.addAll(rightSource.otherPipelineBuilders);
+ }
+
+ public DataStreamAction toFile(String filePath) {
+ FileSink fileChannel = new FileSink(filePath);
+ ChainStage<?> output = mainPipelineBuilder.createStage(fileChannel);
+ mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toPrint() {
+ return toPrint(-1);
+ }
+
+ public DataStreamAction toPrint(int batchSize) {
+ OutputPrintChannel outputPrintChannel = new OutputPrintChannel();
+ if (batchSize > 0) {
+ outputPrintChannel.setBatchSize(batchSize);
+ }
+ ChainStage output = this.mainPipelineBuilder.createStage(outputPrintChannel);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toDB(String url, String userName, String password, String tableName) {
+ DBSink dbChannel = new DBSink(url, userName, password);
+ dbChannel.setTableName(tableName);
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(dbChannel);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toMetaq(String topic) {
+ return toMetaq(topic, null);
+ }
+
+ public DataStreamAction toMetaq(String topic, String tags) {
+ return toMetaq(topic, tags, -1);
+ }
+
+ public DataStreamAction toMetaq(String topic, String tags, int batchSize) {
+ RocketMQSink metaqChannel = new RocketMQSink();
+ metaqChannel.setTopic(topic);
+ metaqChannel.setTags(tags);
+ if (batchSize > 0) {
+ metaqChannel.setBatchSize(batchSize);
+ }
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(metaqChannel);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction toRocketmq(String topic, String groupName, String endpoint, String namesrvAddr,
+ String accessKey, String secretKey, String instanceId) {
+ return toRocketmq(topic, "*", groupName, namesrvAddr, endpoint, accessKey, secretKey, instanceId);
+ }
+
+ public DataStreamAction toRocketmq(String topic, String tags, String groupName, String endpoint,
+ String namesrvAddr, String accessKey, String secretKey, String instanceId) {
+ return toRocketmq(topic, tags, -1, groupName, namesrvAddr, endpoint, accessKey, secretKey, instanceId);
+ }
+
+ public DataStreamAction toRocketmq(String topic, String tags, int batchSize, String groupName,
+ String endpoint, String namesrvAddr, String accessKey, String secretKey, String instanceId) {
+ RocketMQSink rocketMQSink = new RocketMQSink();
+ rocketMQSink.setTopic(topic);
+ rocketMQSink.setTags(tags);
+ rocketMQSink.setGroupName(groupName);
+ rocketMQSink.setNamesrvAddr(namesrvAddr);
+ if (batchSize > 0) {
+ rocketMQSink.setBatchSize(batchSize);
+ }
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(rocketMQSink);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+
+ public DataStreamAction to(ISink<?> sink) {
+ ChainStage<?> output = this.mainPipelineBuilder.createStage(sink);
+ this.mainPipelineBuilder.setTopologyStages(currentChainStage, output);
+ return new DataStreamAction(this.mainPipelineBuilder, this.otherPipelineBuilders, output);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/JoinStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/JoinStream.java
new file mode 100644
index 0000000..26e6420
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/JoinStream.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.common.model.NameCreator;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.function.expression.Equals;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression;
+import org.apache.rocketmq.streams.script.operator.impl.ScriptOperator;
+import org.apache.rocketmq.streams.window.operator.join.JoinWindow;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class JoinStream {
+
+ private static final String INNER_VAR_NAME_PREFIX = "___";
+ protected JoinWindow joinWindow;//完成join 条件的添加
+ protected boolean isDimJoin = false;//是否是维表join
+
+ protected AbstractDim dim;//维度表对象
+ protected String onCondition;//条件
+ protected JoinType joinType;//连接类型
+
+ //用于返回DataStream流
+ protected PipelineBuilder pipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+ /**
+ * 双流join 场景
+ *
+ * @param joinWindow
+ * @param pipelineBuilder
+ * @param pipelineBuilders
+ * @param currentChainStage
+ */
+ public JoinStream(JoinWindow joinWindow, PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ this.joinWindow = joinWindow;
+ }
+
+ /**
+ * 维表join 场景
+ *
+ * @param pipelineBuilder
+ * @param pipelineBuilders
+ * @param currentChainStage
+ */
+ public JoinStream(AbstractDim dim, PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ this.dim = dim;
+ }
+
+ public JoinStream setJoinType(JoinType joinType) {
+ this.joinType = joinType;
+
+ return this;
+ }
+
+ /**
+ * 指定窗口，如果不指定，默认1个小时
+ *
+ * @param time
+ * @return
+ */
+ public JoinStream window(Time time) {
+
+ //维表join 不需要设置
+ if (isDimJoin) {
+ throw new RuntimeException("can not support this method");
+ }
+ joinWindow.setTimeUnitAdjust(1);
+ joinWindow.setSizeInterval(time.getValue());
+ joinWindow.setSlideInterval(time.getValue());
+ joinWindow.setRetainWindowCount(1);
+ return this;
+ }
+
+ /**
+ * 增加条件,用表达式形式表达(leftFieldName,function,rightFieldName)&&({name,==,otherName}||(age,==,age)) 后续再增加结构化的方法
+ *
+ * @param onCondition (leftFieldName,function,rightFieldName)&&({name,==,otherName}||(age,==,age))
+ * @return
+ */
+ public JoinStream setCondition(String onCondition) {
+ this.onCondition = onCondition;
+ return this;
+ }
+
+ public DataStream toDataSteam() {
+ if (isDimJoin) {
+ return doDimJoin();
+ } else {
+ return doJoin();
+ }
+
+ }
+
+ /**
+ * 维度表join的场景
+ */
+ protected DataStream doDimJoin() {
+ String script = null;
+ if (JoinType.INNER_JOIN == joinType) {
+ String data = createName("inner_join");
+ script = data + "=inner_join('" + dim.getNameSpace() + "','" + dim.getConfigureName() + "','" + onCondition + "'," + null + ",''," + null + ");splitArray('" + data + "');";
+ } else if ((JoinType.LEFT_JOIN == joinType)) {
+ String data = createName("left_join");
+ script = data + "=left_join('" + dim.getNameSpace() + "','" + dim.getConfigureName() + "','" + onCondition + "'," + null + ",'" + null + "'," + null + ");if(!null(" + data + ")){splitArray('" + data + "');};";
+ }
+ ChainStage stage = this.pipelineBuilder.createStage(new ScriptOperator(script));
+ this.pipelineBuilder.setTopologyStages(currentChainStage, stage);
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, stage);
+ }
+
+ /**
+ * 双流join的场景
+ */
+ protected DataStream doJoin() {
+ if (JoinType.INNER_JOIN == joinType) {
+ joinWindow.setJoinType("INNER");
+ } else if (JoinType.LEFT_JOIN == joinType) {
+ joinWindow.setJoinType("LEFT");
+ } else {
+ throw new RuntimeException("can not support this join type, expect INNER,LEFT, real is " + joinType.toString());
+ }
+
+ AtomicBoolean hasNoEqualsExpression = new AtomicBoolean(false);//是否有非等值的join 条件
+ Map<String, String> left2Right = createJoinFieldsFromCondition(onCondition, hasNoEqualsExpression);//把等值条件的左右字段映射成map
+ List<String> leftList = new ArrayList<>();
+ List<String> rightList = new ArrayList<>();
+ leftList.addAll(left2Right.keySet());
+ rightList.addAll(left2Right.values());
+ joinWindow.setLeftJoinFieldNames(leftList);
+ joinWindow.setRightJoinFieldNames(rightList);
+ //如果有非等值，则把这个条件设置进去
+ if (hasNoEqualsExpression.get()) {
+ joinWindow.setExpression(onCondition);
+ }
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+
+ /**
+ * 支持的连接类型，目前支持inner join和left join
+ */
+ public enum JoinType {
+ INNER_JOIN,
+ LEFT_JOIN
+ }
+
+ /**
+ * 从条件中找到join 左右的字段。如果有非等值，则不包含在内
+ *
+ * @param
+ * @param onCondition
+ * @return
+ */
+ public Map<String, String> createJoinFieldsFromCondition(String onCondition, AtomicBoolean hasNoEqualsExpression) {
+ List<Expression> expressions = new ArrayList<>();
+ List<RelationExpression> relationExpressions = new ArrayList<>();
+ ExpressionBuilder.createOptimizationExpression("tmp", "tmp", onCondition, expressions, relationExpressions);
+ Map<String, String> left2Right = new HashMap<>();
+ for (Expression expression : expressions) {
+ String varName = expression.getVarName();
+ String valueName = expression.getValue().toString();
+ if (!Equals.isEqualFunction(expression.getFunctionName())) {
+ hasNoEqualsExpression.set(true);
+ continue;
+ }
+ left2Right.put(varName, valueName);
+ }
+ return left2Right;
+ }
+
+ public static String createName(String functionName, String... names) {
+ if (names == null || names.length == 0) {
+ return NameCreator.createNewName(INNER_VAR_NAME_PREFIX, functionName);
+ }
+ String[] values = new String[names.length + 2];
+ values[0] = INNER_VAR_NAME_PREFIX;
+ values[1] = functionName;
+ for (int i = 2; i < values.length; i++) {
+ values[i] = names[i - 2];
+ }
+ return NameCreator.createNewName(values);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/SplitStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/SplitStream.java
new file mode 100644
index 0000000..49efcd2
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/SplitStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import org.apache.rocketmq.streams.common.context.AbstractContext;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.udf.StageBuilder;
+
+import java.util.Set;
+
+public class SplitStream {
+
+
+ /**
+ * 创建datastream时使用
+ */
+ protected PipelineBuilder pipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+
+ public SplitStream(PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ }
+
+ /**
+ * 选择一个分支
+ * @param lableName
+ * @return
+ */
+ public DataStream select(String lableName){
+ StageBuilder stage = new StageBuilder() {
+ @Override
+ protected <T> T operate(IMessage message, AbstractContext context) {
+ return null;
+ }
+ };
+ stage.setLabel(lableName);
+ this.pipelineBuilder.setTopologyStages(currentChainStage,stage);
+ return new DataStream(pipelineBuilder,otherPipelineBuilders,stage);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/WindowStream.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/WindowStream.java
new file mode 100644
index 0000000..1f6ea24
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/WindowStream.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.rocketmq.streams.common.context.UserDefinedMessage;
+import org.apache.rocketmq.streams.common.functions.ReduceFunction;
+import org.apache.rocketmq.streams.common.topology.ChainStage;
+import org.apache.rocketmq.streams.common.topology.builder.PipelineBuilder;
+import org.apache.rocketmq.streams.common.topology.stages.udf.IReducer;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+
+import java.util.Set;
+
+/**
+ * 做windown 相关操作 可以同时设置多个统计算子，如count，sum，avg 通过toDataSteam/reduce 返回DataSteam
+ */
+public class WindowStream {
+ //window 对象
+ protected AbstractWindow window;
+
+ /**
+ * 创建datastream时使用
+ */
+ protected PipelineBuilder pipelineBuilder;
+ protected Set<PipelineBuilder> otherPipelineBuilders;
+ protected ChainStage<?> currentChainStage;
+
+ public WindowStream(AbstractWindow window, PipelineBuilder pipelineBuilder, Set<PipelineBuilder> pipelineBuilders, ChainStage<?> currentChainStage) {
+ this.pipelineBuilder = pipelineBuilder;
+ this.otherPipelineBuilders = pipelineBuilders;
+ this.currentChainStage = currentChainStage;
+ this.window = window;
+ }
+
+ /**
+ * 做count算子
+ *
+ * @param asName count结果对应的名字，如 sql中count(1) as c 。asName=c
+ * @return
+ */
+ public WindowStream count(String asName) {
+ window.getSelectMap().put(asName, asName + "=count(" + asName + ")");
+ return this;
+ }
+
+ /**
+ * 做min算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @return
+ */
+ public WindowStream min(String fieldName) {
+ window.getSelectMap().put(fieldName, fieldName + "=min(" + fieldName + ")");
+ return this;
+ }
+
+ /**
+ * 做max算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @return
+ */
+ public WindowStream max(String fieldName) {
+ window.getSelectMap().put(fieldName, fieldName + "=max(" + fieldName + ")");
+ return this;
+ }
+
+ /**
+ * 做avg算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @param asName avg结果对应的名字，如 sql中avg(name) as c 。asName=c
+ * @return
+ */
+ public WindowStream avg(String fieldName, String asName) {
+ window.getSelectMap().put(asName, asName + "=avg(" + fieldName + ")");
+ return this;
+ }
+
+ /**
+ * 做sum算子
+ *
+ * @param fieldName 算子需要操作的字段名
+ * @param asName sum结果对应的名字，如 sql中sum(name) as c 。asName=c
+ * @return
+ */
+ public WindowStream sum(String fieldName, String asName) {
+ window.getSelectMap().put(asName, asName + "=sum(" + fieldName + ")");
+ return this;
+ }
+
+ public WindowStream setTimeField(String timeField) {
+ window.setTimeFieldName(timeField);
+ return this;
+ }
+
+ public WindowStream setFireMode(int fireMode) {
+ window.setFireMode(fireMode);
+ return this;
+ }
+
+ public WindowStream setLocalStorageOnly(boolean isLocalStorageOnley) {
+ window.setLocalStorageOnly(isLocalStorageOnley);
+ return this;
+ }
+
+ public WindowStream setMaxMsgGap(Long maxMsgGapSecond) {
+ window.setMsgMaxGapSecond(maxMsgGapSecond);
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param fieldNames
+ * @return
+ */
+ public WindowStream groupBy(String... fieldNames) {
+ window.setGroupByFieldName(MapKeyUtil.createKeyBySign(";", fieldNames));
+ for (String fieldName : fieldNames) {
+ window.getSelectMap().put(fieldName, fieldName);
+ }
+
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param fireMode
+ * @return
+ */
+ public WindowStream fireMode(int fireMode) {
+ window.setFireMode(fireMode);
+
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param waterMarkSecond
+ * @return
+ */
+ public WindowStream waterMark(int waterMarkSecond) {
+ window.setWaterMarkMinute(waterMarkSecond);
+
+ return this;
+ }
+
+ /**
+ * 以哪几个字段做分组，支持多个字段
+ *
+ * @param fieldName
+ * @return
+ */
+ public WindowStream timeField(String fieldName) {
+ window.setTimeFieldName(fieldName);
+
+ return this;
+ }
+
+ /**
+ * 用户自定义reduce逻辑
+ *
+ * @param reduceFunction
+ * @return
+ */
+ public <R, O> DataStream reduce(ReduceFunction<R, O> reduceFunction) {
+ window.setReducer(new IReducer() {
+ @Override
+ public JSONObject reduce(JSONObject accumulator, JSONObject msg) {
+ Object accumulatorValue = accumulator;
+ Object msgValue = msg;
+ if (msg instanceof UserDefinedMessage) {
+ UserDefinedMessage userDefinedMessage = (UserDefinedMessage)msg;
+ msgValue = userDefinedMessage.getMessageValue();
+ }
+ if (accumulator instanceof UserDefinedMessage) {
+ UserDefinedMessage userDefinedMessage = (UserDefinedMessage)accumulator;
+ accumulatorValue = userDefinedMessage.getMessageValue();
+ }
+ R result = reduceFunction.reduce((R)accumulatorValue, (O)msgValue);
+ return new UserDefinedMessage(result);
+ }
+ });
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+
+ public DataStream toDataSteam() {
+ return new DataStream(pipelineBuilder, otherPipelineBuilders, currentChainStage);
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/HoppingWindow.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/HoppingWindow.java
new file mode 100644
index 0000000..0c945f0
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/HoppingWindow.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+public class HoppingWindow {
+ /**
+ * 滑动窗口信息
+ * @return
+ */
+ public static WindowInfo of(Time windowSize,Time windowSlide){
+ WindowInfo windowInfo=new WindowInfo();
+ windowInfo.setType(WindowInfo.HOPPING_WINDOW);
+ windowInfo.setWindowSize(windowSize);
+ windowInfo.setWindowSlide(windowSlide);
+ return windowInfo;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/SessionWindow.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/SessionWindow.java
new file mode 100644
index 0000000..67c7f22
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/SessionWindow.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+public class SessionWindow {
+ /**
+ * 滑动窗口信息
+ * @param time
+ * @return
+ */
+ public static WindowInfo of(Time time){
+ WindowInfo windowInfo=new WindowInfo();
+ windowInfo.setType(WindowInfo.SESSION_WINDOW);
+ windowInfo.setWindowSize(time);
+ return windowInfo;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/Time.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/Time.java
new file mode 100644
index 0000000..2157e01
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/Time.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+/**
+ * 给window指定窗口时间
+ */
+public class Time {
+ protected int value;
+ public Time(int value){
+ this.value=value;
+ }
+ public static Time seconds(int second){
+
+ return new Time(second);
+ }
+ public static Time minutes(int minutes){
+ return new Time(minutes*60);
+ }
+ public static Time hours(int hours){
+ return new Time(60*60*hours);
+ }
+ public static Time days(int days){
+ return new Time(60*60*24*days);
+ }
+
+ public int getValue() {
+ return value;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/TumblingWindow.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/TumblingWindow.java
new file mode 100644
index 0000000..62c0ce3
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/TumblingWindow.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client.transform.window;
+
+public class TumblingWindow {
+ /**
+ * 滑动窗口信息
+ *
+ * @param time
+ * @return
+ */
+ public static WindowInfo of(Time time) {
+ WindowInfo windowInfo = new WindowInfo();
+ windowInfo.setType(WindowInfo.TUMBLING_WINDOW);
+ windowInfo.setWindowSize(time);
+ return windowInfo;
+ }
+}
diff --git a/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
new file mode 100644
index 0000000..64be503
--- /dev/null
+++ b/rocketmq-streams-clients/src/main/java/org/apache/rocketmq/streams/client/transform/window/WindowInfo.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.client.transform.window;
+
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+import org.apache.rocketmq.streams.window.operator.impl.SessionWindow;
+import org.apache.rocketmq.streams.window.operator.impl.WindowOperator;
+
+/**
+ * 保存创建window的信息 主要是窗口类型，窗口大小
+ */
+public class WindowInfo {
+ public static int HOPPING_WINDOW = 1;//滑动窗口
+ public static int TUMBLING_WINDOW = 2;//滚动窗口
+ public static int SESSION_WINDOW = 23;
+ protected int type;//window类型 hopping，Tumbling
+ protected Time windowSize;//窗口大小
+ protected Time windowSlide;//滑动大小
+
+ /**
+ * 创建窗口
+ *
+ * @return
+ */
+ public AbstractWindow createWindow() {
+ AbstractWindow window = null;
+ if (type == HOPPING_WINDOW) {
+ window = new WindowOperator();
+ window.setTimeUnitAdjust(1);
+ window.setSizeInterval(windowSize.getValue());
+ window.setSlideInterval(windowSlide.getValue());
+ } else if (type == TUMBLING_WINDOW) {
+ window = new WindowOperator();
+ window.setTimeUnitAdjust(1);
+ window.setSizeInterval(windowSize.getValue());
+ } else if (type == SESSION_WINDOW) {
+ window = new SessionWindow();
+ window.setTimeUnitAdjust(1);
+ window.setSizeInterval(windowSize.getValue());
+ } else {
+ throw new RuntimeException("can not support the type ,expect 1，2，3。actual is " + type);
+ }
+ return window;
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public void setType(int type) {
+ this.type = type;
+ }
+
+ public Time getWindowSize() {
+ return windowSize;
+ }
+
+ public void setWindowSize(Time windowSize) {
+ this.windowSize = windowSize;
+ }
+
+ public Time getWindowSlide() {
+ return windowSlide;
+ }
+
+ public void setWindowSlide(Time windowSlide) {
+ this.windowSlide = windowSlide;
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
new file mode 100644
index 0000000..7e078f8
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DBDriverTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.configuable.ConfigurableComponent;
+import org.apache.rocketmq.streams.configuable.model.Configure;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * 数据库的存储，需要配置存储的连接参数，请先完成配置，后执行单元用例 如果未建表，可以通过Configure.createTableSQL() 获取建表语句，创建表后，测试
+ */
+public class DBDriverTest {
+ private String URL = "";
+ protected String USER_NAME = "";
+ protected String PASSWORD = "";
+ protected String TABLE_NAME = "rocketmq_streams_configure_source";
+
+ @Test
+ public void testDBConfigurableService() {
+ String namespace = "streams.db.configurable";
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE, "DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_TABLE_NAME, TABLE_NAME);
+
+ //如果表不存在，创建表
+ String sql = (Configure.createTableSQL(TABLE_NAME));
+ DriverBuilder.createDriver().execute(sql);
+ ConfigurableComponent configurableComponent = ConfigurableComponent.getInstance(namespace);
+ configurableComponent.insert(createPerson(namespace));
+ configurableComponent.refreshConfigurable(namespace);
+ Person person = configurableComponent.queryConfigurable("person", "peronName");
+ assertNotNull(person);
+ }
+
+ /**
+ * 创建configuable对象
+ *
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setName("chris");
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DataStreamTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DataStreamTest.java
new file mode 100644
index 0000000..8f54067
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/DataStreamTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import org.apache.rocketmq.streams.client.source.DataStreamSource;
+import org.apache.rocketmq.streams.client.strategy.CheckpointStrategy;
+import org.apache.rocketmq.streams.client.strategy.StateStrategy;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.sql.*;
+
+public class DataStreamTest implements Serializable {
+
+ DataStreamSource dataStream;
+
+ @Before
+ public void init() {
+ dataStream = StreamBuilder.dataStream("test_namespace", "graph_pipeline");
+ }
+
+ @Test
+ public void testFromFile() {
+ dataStream
+ .fromFile("/Users/junjie.cheng/text.txt", false)
+ .map(message -> message + "--")
+ .toPrint(1)
+ .start();
+ }
+
+ @Test
+ public void testRocketmq() {
+ dataStream
+ .fromRocketmq("TOPIC_EVENT_SAS_SECURITY_EVENT", "111")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .start();
+ }
+
+ @Test
+ public void testDBCheckPoint() {
+ dataStream
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("", "", "", 0L))
+ .start();
+ }
+
+ @Test
+ public void testFileCheckPoint() {
+ dataStream
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.mem(0L))
+ .start();
+ }
+
+ @Test
+ public void testBothStrategy() {
+ dataStream
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.mem(0L), StateStrategy.db())
+ .start();
+ }
+
+ @Test
+ public void testMeta() {
+ try {
+ Class.forName("com.mysql.jdbc.Driver");
+ Connection connection = DriverManager.getConnection("", "", "");
+ DatabaseMetaData metaData = connection.getMetaData();
+ ResultSet dataFilter = metaData.getColumns(connection.getCatalog(), "%", "XXX", null);
+ while (dataFilter.next()) {
+ String one = dataFilter.getString("DATA_TYPE");
+ int two = dataFilter.getInt("COLUMN_SIZE");
+ String three = dataFilter.getString("COLUMN_NAME");
+ String four = dataFilter.getString("TYPE_NAME");
+ System.out.println(one + " " + two + " " + three + " " + four + " " + DataTypeUtil.getDataType(dataFilter.getString("TYPE_NAME")));
+ }
+
+ } catch (ClassNotFoundException | SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/FilterTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/FilterTest.java
new file mode 100644
index 0000000..19665fd
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/FilterTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.util.List;
+import org.apache.rocketmq.streams.filter.FilterComponent;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+public class FilterTest {
+ @Test
+ public void testFilter(){
+ JSONObject msg=new JSONObject();
+ msg.put("name","chris");
+ msg.put("age",18);
+ Rule rule= ExpressionBuilder.createRule("tmp","tmp","(name,==,chris)&(age,>=,18)");
+ FilterComponent filterComponent=FilterComponent.getInstance();
+ List<Rule> fireRules=filterComponent.excuteRule(msg,rule);
+ assertTrue(fireRules.size()==1);
+ }
+
+ @Test
+ public void testFilter2(){
+ JSONObject msg=new JSONObject();
+ msg.put("name","chris");
+ msg.put("age",18);
+ boolean result= ExpressionBuilder.executeExecute("tmp","(name,==,chris)&(age,>,int,18)",msg);
+ assertTrue(!result);
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/JoinTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/JoinTest.java
new file mode 100644
index 0000000..b221e92
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/JoinTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.JoinStream.JoinType;
+import org.apache.rocketmq.streams.common.functions.FilterFunction;
+import org.junit.Test;
+
+public class JoinTest implements Serializable {
+
+ @Test
+ public void testJoin() {
+ DataStream leftStream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ DataStream rightStream = (StreamBuilder.dataStream("namespace", "name2")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ leftStream.join(rightStream).setJoinType(JoinType.INNER_JOIN)
+ .setCondition("(ProjectName,==,ProjectName)&(LogStore,==,LogStore)")
+ .toDataSteam()
+ .toPrint()
+ .start();
+
+ }
+
+ @Test
+ public void testDim() {
+ DataStreamAction stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }))
+ .join("dburl", "dbUserName", "dbPassowrd", "tableNameOrSQL", 5)
+ .setCondition("(name,==,name)")
+ .toDataSteam()
+ .selectFields("name", "age", "address")
+ .toPrint();
+ stream.start();
+ ;
+ ;
+
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/LeaseTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/LeaseTest.java
new file mode 100644
index 0000000..1d3ec8a
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/LeaseTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import java.util.Date;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.lease.LeaseComponent;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class LeaseTest {
+
+ private String URL="";
+ protected String USER_NAME="";
+ protected String PASSWORD="";
+
+ public LeaseTest(){
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE,"DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL,URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME,USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD,PASSWORD);//password
+
+ /**
+ * 创建lease info表
+ */
+ JDBCDriver driver= DriverBuilder.createDriver();
+ driver.execute(LeaseInfo.createTableSQL());
+ }
+
+ @Test
+ public void testLease() throws InterruptedException {
+ String leaseName="lease.test";
+ int leaseTime=5;
+ LeaseComponent.getInstance().getService().startLeaseTask(leaseName, leaseTime, new ILeaseGetCallback() {
+ @Override
+ public void callback(Date nextLeaseDate) {
+ System.out.println("I get lease");
+ }
+ });
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ }
+
+ @Test
+ public void testLock() throws InterruptedException {
+ String name="dipper";
+ String lockName="lease.test";
+ int leaseTime=5;
+ boolean success=LeaseComponent.getInstance().getService().lock(name,lockName,leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(6000);
+ assertTrue(!LeaseComponent.getInstance().getService().hasHoldLock(name,lockName));//超期释放
+ }
+
+ /**
+ * holdlock是一直持有锁，和租约的区别是，当释放锁后，无其他实例抢占
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testHoldLock() throws InterruptedException {
+ String name="dipper";
+ String lockName="lease.test";
+ int leaseTime=6;
+ boolean success=LeaseComponent.getInstance().getService().holdLock(name,lockName,leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(8000);
+ assertTrue(LeaseComponent.getInstance().getService().hasHoldLock(name,lockName));//会自动续约，不会释放，可以手动释放
+ LeaseComponent.getInstance().getService().unlock(name,lockName);
+ assertTrue(!LeaseComponent.getInstance().getService().hasHoldLock(name,lockName));
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
new file mode 100644
index 0000000..a8ba9ab
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/ORMUtilTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.junit.Test;
+
+public class ORMUtilTest {
+ private String URL="";
+ protected String USER_NAME="";
+ protected String PASSWORD="";
+
+ public ORMUtilTest(){
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL,URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME,USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD,PASSWORD);//password
+ }
+ @Test
+ public void testInsert(){
+ String namespace="org.apache.configuable.test";
+ List<Person> personList=new ArrayList<>();
+ for(int i=0;i<10;i++){
+ personList.add(createPerson(namespace,"chris"+i));
+ }
+ /**
+ * 不带数据库连接信息（url,userName,Password），使用ConfiguableComponet的连接信息
+ */
+ ORMUtil.batchIgnoreInto(personList);//批量插入，如果有唯一键冲突，替换
+ ORMUtil.batchIgnoreInto(personList);//批量插入，如果有唯一键冲突，忽略
+ ORMUtil.batchInsertInto(personList);////批量插入，如果有唯一键冲突，跑错
+ }
+
+ @Test
+ public void testQueryList(){
+ Map<String,Integer> paras=new HashMap<>();
+ paras.put("age",18);
+ List<Person> personList=ORMUtil.queryForList("select * from person where age >${age} limit 100",paras,Person.class);
+ }
+
+ @Test
+ public void testQueryOneRow(){
+ Person personPara=new Person();
+ personPara.setAge(18);
+ personPara.setName("chris1");
+ Person person=ORMUtil.queryForObject("select * from person where age =${age} and name='${name}' ",personPara,Person.class,URL,USER_NAME,PASSWORD);
+ }
+
+ /**
+ * 创建configuable对象
+ * @param namespace
+ * @return
+ */
+ protected Person createPerson(String namespace,String name){
+ Person person=new Person();
+ person.setName(name);
+ person.setAge(18);
+ person.setNameSpace(namespace);
+ person.setConfigureName("peronName");
+ person.setType("person");
+ return person;
+ }
+}
+
+
+class Person extends BasedConfigurable {
+ @ENVDependence
+ private String name;
+ private int age;
+ private Boolean isMale;
+ private List<String> addresses;
+ private Map<String, Integer> childName2Age;
+
+ public static Person createPerson(String namespace) {
+ Person person = new Person();
+ person.setNameSpace(namespace);
+ person.setType("person");
+ person.setConfigureName("Chris");
+ person.setName("Chris");
+ List<String> addresses = new ArrayList<>();
+ addresses.add("huilongguan");
+ addresses.add("shangdi");
+ person.setAddresses(addresses);
+ Map<String, Integer> childName2Age = new HashMap<>();
+ childName2Age.put("yuanyahan", 8);
+ childName2Age.put("yuanruxi", 4);
+ person.setChildName2Age(childName2Age);
+ person.setMale(true);
+ person.setAge(18);
+ return person;
+ }
+
+ @Override
+ public String toString() {
+ return "org.apache.rocketmq.streams.Person{" + "name='" + name + '\'' + ", age=" + age + ", isMale=" + isMale + ", addresses=" + addresses
+ + ", childName2Age=" + childName2Age + '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public Boolean getMale() {
+ return isMale;
+ }
+
+ public void setMale(Boolean male) {
+ isMale = male;
+ }
+
+ public List<String> getAddresses() {
+ return addresses;
+ }
+
+ public void setAddresses(List<String> addresses) {
+ this.addresses = addresses;
+ }
+
+ public Map<String, Integer> getChildName2Age() {
+ return childName2Age;
+ }
+
+ public void setChildName2Age(Map<String, Integer> childName2Age) {
+ this.childName2Age = childName2Age;
+ }
+
+ @Override
+ public Object clone() {
+ Person person = null;
+ try {
+ person = (Person)super.clone();
+ } catch (CloneNotSupportedException e) {
+ System.out.println("clone error " + e);
+ }
+ return person;
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/SplitTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/SplitTest.java
new file mode 100644
index 0000000..56ab5ae
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/SplitTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.SplitStream;
+import org.apache.rocketmq.streams.common.functions.FilterFunction;
+import org.apache.rocketmq.streams.common.functions.SplitFunction;
+import org.junit.Test;
+
+public class SplitTest implements Serializable {
+
+ @Test
+ public void testUnion() {
+ DataStream stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ SplitStream splitStream = stream.split(new SplitFunction<JSONObject>() {
+ @Override
+ public String split(JSONObject o) {
+ if (o.getInteger("age") < 18) {
+ return "children";
+ } else if (o.getInteger("age") >= 18) {
+ return "adult";
+ }
+ return null;
+ }
+ });
+
+ DataStream children = splitStream.select("children");
+ DataStream adult = splitStream.select("adult");
+ children.union(adult)
+ .toPrint()
+ .start();
+
+ }
+
+ @Test
+ public void testDim() {
+ DataStreamAction stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }))
+ .join("dburl", "dbUserName", "dbPassowrd", "tableNameOrSQL", 5)
+ .setCondition("(name,==,name)")
+ .toDataSteam()
+ .selectFields("name", "age", "address")
+ .toPrint();
+
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/UnionTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/UnionTest.java
new file mode 100644
index 0000000..1b49185
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/UnionTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+import java.io.Serializable;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.functions.FilterFunction;
+import org.junit.Test;
+
+public class UnionTest implements Serializable {
+
+ @Test
+ public void testUnion() {
+ DataStream leftStream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ DataStream rightStream = (StreamBuilder.dataStream("namespace", "name2")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }));
+
+ leftStream.union(rightStream).toPrint().start();
+ }
+
+ @Test
+ public void testDim() {
+ DataStreamAction stream = (StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_1000.txt")
+ .filter(new FilterFunction<JSONObject>() {
+
+ @Override
+ public boolean filter(JSONObject value) throws Exception {
+ if (value.getString("ProjectName") == null || value.getString("LogStore") == null) {
+ return true;
+ }
+ return false;
+ }
+ }))
+ .join("dburl", "dbUserName", "dbPassowrd", "tableNameOrSQL", 5)
+ .setCondition("(name,==,name)")
+ .toDataSteam()
+ .selectFields("name", "age", "address")
+ .toPrint();
+
+ stream.start();
+
+ }
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/WindowTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/WindowTest.java
new file mode 100644
index 0000000..69d98b2
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/WindowTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.client;
+
+import com.alibaba.fastjson.JSONObject;
+
+import java.io.Serializable;
+
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.client.transform.window.TumblingWindow;
+import org.apache.rocketmq.streams.common.functions.ForEachFunction;
+import org.apache.rocketmq.streams.common.functions.MapFunction;
+import org.junit.Test;
+
+public class WindowTest implements Serializable {
+
+ @Test
+ public void testWindow() {
+ StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/duheng/project/opensource/sls_100.txt", false)
+ .map((MapFunction<JSONObject, String>)message -> JSONObject.parseObject(message))
+ .window(TumblingWindow.of(Time.seconds(5)))
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(true)
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "InFlow")
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+ protected int sum = 0;
+
+ @Override
+ public void foreach(JSONObject o) {
+ int total = o.getInteger("total");
+ sum = sum + total;
+ o.put("sum(total)", sum);
+ }
+ }).toPrint().start();
+
+ }
+
+ // @Test
+ // public void testWindowFromMetaq() throws InterruptedException {
+ // String topic = "TOPIC_DIPPER_SYSTEM_MSG_4";
+ // StreamBuilder.dataStream("namespace", "name")
+ // .fromFile("/Users/yuanxiaodong/chris/sls_100.txt", true)
+ // .toRocketmq(topic)
+ // .asyncStart();
+ //
+ // StreamBuilder.dataStream("namespace", "name1")
+ // .fromRocketmq(topic, "chris", true)
+ // .window(TumblingWindow.of(Time.seconds(5)))
+ // .groupby("ProjectName", "LogStore")
+ // .setLocalStorageOnly(true)
+ // .count("total")
+ // .sum("OutFlow", "OutFlow")
+ // .sum("InFlow", "inflow")
+ // .toDataSteam()
+ // .forEach(new ForEachFunction<JSONObject>() {
+ // protected int sum = 0;
+ //
+ // @Override
+ // public void foreach(JSONObject o) {
+ // int total = o.getInteger("total");
+ // sum = sum + total;
+ // o.put("sum(total)", sum);
+ // }
+ // }).toPrint().start();
+ // }
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/AbstractWindowFireModeTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/AbstractWindowFireModeTest.java
new file mode 100644
index 0000000..c102f1f
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/AbstractWindowFireModeTest.java
@@ -0,0 +1,189 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.client.transform.window.Time;
+import org.apache.rocketmq.streams.client.transform.window.TumblingWindow;
+import org.apache.rocketmq.streams.common.functions.ForEachFunction;
+import org.apache.rocketmq.streams.common.functions.MapFunction;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+
+
+public abstract class AbstractWindowFireModeTest implements Serializable {
+ protected Date date=new Date();
+ public AbstractWindowFireModeTest(){
+
+ date.setYear(2021-1900);
+ date.setMonth(6);
+ date.setDate(14);
+ date.setHours(12);
+ date.setMinutes(1);
+ date.setSeconds(0);
+
+ }
+
+ public void testWindowFireMode0(boolean isLocalOnly) throws InterruptedException {
+ testWindowFireMode1(isLocalOnly,5);
+ }
+
+ public void testWindowFireMode0(boolean isLocalOnly,int windowSize) throws InterruptedException {
+
+ createSourceDataStream().map(new MapFunction<JSONObject, String>() {
+ int count=0;
+ Long time=null;
+ @Override
+ public JSONObject map(String message) throws Exception {
+
+ if(time==null){
+ time=date.getTime();
+ }else {
+ time+=count;
+ }
+ count++;
+ JSONObject msg=JSONObject.parseObject(message);
+
+ msg.put("logTime",time);
+
+ return msg;
+ }
+ })
+ .window(TumblingWindow.of(Time.seconds(windowSize)))
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(isLocalOnly)
+ .setMaxMsgGap(isLocalOnly?10L:20L)
+ .setTimeField("logTime")
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "inflow")
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+ AtomicInteger sum = new AtomicInteger(0) ;
+ @Override
+ public synchronized void foreach(JSONObject o) {
+ int total = o.getInteger("total");
+ o.put("sum(total)", sum.addAndGet(total));
+ }
+ }).toPrint().start();
+ }
+ public void testWindowFireMode1(boolean isLocalOnly) throws InterruptedException {
+ testWindowFireMode1(isLocalOnly,5);
+ }
+ public void testWindowFireMode1(boolean isLocalOnly,int windowSize) throws InterruptedException {
+ AtomicInteger sum = new AtomicInteger(0) ;
+ createSourceDataStream()
+ //.map(new MapFunction<JSONObject, String>() {
+ // AtomicInteger COUNT=new AtomicInteger(0);
+ // Long time;
+ // @Override
+ // public JSONObject map(String message) throws Exception {
+ //
+ // if(time==null){
+ // time=date.getTime();
+ // }else {
+ // int count=COUNT.incrementAndGet();
+ // time+=count;
+ // }
+ // JSONObject msg=JSONObject.parseObject(message);
+ //
+ // msg.put("logTime",time);
+ // return msg;
+ // }
+ //})
+ .window(TumblingWindow.of(Time.seconds(windowSize)))
+ .setTimeField("logTime")
+ .fireMode(1)
+ .setMaxMsgGap(isLocalOnly?20L:20L)
+ .waterMark(100000000)
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(isLocalOnly)
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "InFlow")
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+
+
+ @Override
+ public synchronized void foreach(JSONObject o) {
+ int total = o.getInteger("total");
+ o.put("sum(total)", sum.addAndGet(total));
+ }
+ }).toPrint().start();
+ }
+
+ public void testWindowFireMode2(boolean isLocalOnly){
+ long time=new Date().getTime();
+ System.out.println(DateUtil.getCurrentTimeString());
+ createSourceDataStream()
+ .map(new MapFunction<JSONObject, String>() {
+ int count=0;
+ @Override
+ public JSONObject map(String message) throws Exception {
+
+ JSONObject msg=JSONObject.parseObject(message);
+ long time= msg.getLong("logTime");
+ Date date=new Date(time);
+ date.setYear(2021-1900);
+ date.setMonth(6);
+ date.setDate(14);
+ msg.put("logTime",date.getTime()+count++);
+ return msg;
+ }
+ })
+ .window(TumblingWindow.of(Time.seconds(5)))
+ .setTimeField("logTime")
+ .setMaxMsgGap(isLocalOnly?5L:20L)
+ .fireMode(1)
+ .waterMark(100000000)
+ .groupBy("ProjectName", "LogStore")
+ .setLocalStorageOnly(isLocalOnly)
+ .count("total")
+ .sum("OutFlow", "OutFlow")
+ .sum("InFlow", "InFlow")
+ .toDataSteam()
+ .map(new MapFunction<JSONObject, JSONObject>() {
+ long time=new Date().getTime();
+ @Override
+ public JSONObject map(JSONObject message) throws Exception {
+ message.put("name","chris");
+ message.put("time",time++);
+ return message;
+ }
+ })
+ .window(TumblingWindow.of(Time.seconds(5)))
+ .fireMode(2).waterMark(100000000)
+ .setMaxMsgGap(80L)
+ .groupBy("name")
+ .setTimeField("time")
+ .sum("total","sum_total")
+ .setLocalStorageOnly(true)
+ .toDataSteam()
+ .forEach(new ForEachFunction<JSONObject>() {
+ AtomicInteger sum = new AtomicInteger(0) ;
+ Map<String,Integer> map=new HashMap<>();
+ @Override
+ public synchronized void foreach(JSONObject o) {
+ String windowInstanceId=o.getString("windowInstanceId");
+ Integer oldValue=map.get(windowInstanceId);
+ int total = o.getInteger("sum_total");
+ if(oldValue!=null){
+ total=total-oldValue;
+ }
+ int nowValue=sum.addAndGet(total);
+ map.put(windowInstanceId,total);
+ o.put("sum(total)", nowValue);
+ }
+ }).toPrint().start();
+
+ }
+
+
+ protected abstract DataStream createSourceDataStream();
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromFileTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromFileTest.java
new file mode 100644
index 0000000..71a2593
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromFileTest.java
@@ -0,0 +1,158 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.PrintUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.batchloader.IRowOperator;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.apache.rocketmq.streams.window.model.WindowInstance;
+import org.apache.rocketmq.streams.window.operator.AbstractWindow;
+import org.apache.rocketmq.streams.window.operator.impl.WindowOperator;
+import org.apache.rocketmq.streams.window.state.impl.WindowValue;
+import org.apache.rocketmq.streams.window.storage.WindowStorage;
+import org.junit.Test;
+
+public class WindowFromFileTest extends AbstractWindowFireModeTest {
+ protected DataStream createSourceDataStream(){
+ return StreamBuilder.dataStream("namespace", "name1")
+ .fromFile("/Users/yuanxiaodong/chris/sls_100000.txt",false);
+ }
+
+ /**
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWindowFireMode0AndNoTimeField() throws InterruptedException {
+ super.testWindowFireMode0(false);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode1() throws InterruptedException {
+ super.testWindowFireMode1(false);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode2() {
+ super.testWindowFireMode2(false);
+ }
+
+
+
+ @Test
+ public void testSLS10(){
+ Map<String,Integer> map=create();
+
+ System.out.println(map.size());
+ PrintUtil.print(map);
+ }
+
+
+ @Test
+ public void testLoadData() throws InterruptedException {
+ Map<String,Integer> map=create();
+ WindowOperator windowOperator=new WindowOperator();
+ windowOperator.setNameSpace("namespace");
+ windowOperator.setConfigureName("name1");
+ WindowInstance windowInstance=windowOperator.createWindowInstance("2021-07-14 02:00:00","2021-07-14 03:00:00","2021-07-14 03:00:00","1");
+ WindowStorage storage= new WindowStorage();
+ storage.loadSplitData2Local("1", windowInstance.createWindowInstanceId(), WindowValue.class, new IRowOperator() {
+ @Override
+ public void doProcess(Map<String, Object> row) {
+ WindowValue windowValue = ORMUtil.convert(row, WindowValue.class);
+ String groupBy=windowValue.getGroupBy();
+ if(!map.containsKey(groupBy)){
+
+ System.out.println(groupBy+" ======");
+ }else {
+ Integer count=(Integer)windowValue.getComputedColumnResultByKey("total");
+ int mapCount=map.get(groupBy);
+ if(count!=mapCount){
+ System.out.println(groupBy+" ======");
+ }
+ }
+ }
+ });
+
+
+ Thread.sleep(10000000l);
+ }
+
+
+ protected Map<String,Integer> create(){
+ List<String> lines= FileUtil.loadFileLine("/Users/yuanxiaodong/chris/sls_10000.txt");
+ Map<String,Integer> map=new HashMap<>();
+ for(String line:lines){
+ JSONObject msg=JSONObject.parseObject(line);
+ String projectName=msg.getString("ProjectName");
+ String logstore=msg.getString("LogStore");
+
+ String key= MapKeyUtil.createKey(projectName,logstore);
+
+ if (StringUtil.isEmpty(key)) {
+ key="<null>";
+ }
+ Integer count=map.get(key);
+ if(count==null){
+ map.put(key,1);
+ }else {
+ count++;
+ map.put(key,count);
+ }
+ }
+ return map;
+ }
+
+
+ @Test
+ public void testFileResult(){
+ createFile(3);
+ }
+
+
+ protected void createFile(int count){
+
+ date.setYear(2021-1900);
+ date.setMonth(6);
+ date.setDate(27);
+ date.setHours(12);
+ date.setMinutes(1);
+ date.setSeconds(0);
+ Long time=null;
+ for(int i=0;i<count;i++){
+ List<String> lines= FileUtil.loadFileLine("/Users/yuanxiaodong/chris/sls_100000.txt");
+ List<String> msgs=new ArrayList<>();
+ for(String line:lines){
+ JSONObject msg=JSONObject.parseObject(line);
+ if(time==null){
+ time=date.getTime();
+ }else {
+ time=time+1;
+ }
+ msg.put("logTime",time);
+ AbstractWindow window=new WindowOperator();
+ window.setSizeInterval(5);
+ window.setTimeUnitAdjust(1);
+ window.setTimeFieldName("logTime");
+ //WindowInstance windowInstance= window.queryOrCreateWindowInstance(new Message(msg),"1").get(0);
+ msgs.add(msg.toJSONString());
+ }
+ FileUtil.write("/Users/yuanxiaodong/chris/sls_100000_time_"+i+".txt",msgs);
+ }
+ }
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromMetaq.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromMetaq.java
new file mode 100644
index 0000000..235ce3d
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowFromMetaq.java
@@ -0,0 +1,47 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.junit.Test;
+
+public class WindowFromMetaq extends AbstractWindowFireModeTest {
+
+ String topic = "TOPIC_DIPPER_SYSTEM_MSG_5";
+ @Test
+ public void testWindowFireMode0() throws InterruptedException {
+ super.testWindowFireMode0(true);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode1() throws InterruptedException {
+ super.testWindowFireMode1(true);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode2() {
+ super.testWindowFireMode2(true);
+ }
+
+
+ @Test
+ public void testWindowToMetaq() throws InterruptedException {
+
+ long start=System.currentTimeMillis();
+ StreamBuilder.dataStream("namespace", "name")
+ .fromFile("/Users/yuanxiaodong/chris/sls_10.txt", true)
+ .toMetaq(topic)
+ .start();
+ }
+
+
+ protected DataStream createSourceDataStream(){
+ return StreamBuilder.dataStream("namespace", "name1")
+ .fromRocketmq(topic,"chris1",true);
+ }
+
+
+}
diff --git a/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowHighAvailabilityTest.java b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowHighAvailabilityTest.java
new file mode 100644
index 0000000..cbca7bb
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/java/org/apache/rocketmq/streams/client/windows/WindowHighAvailabilityTest.java
@@ -0,0 +1,131 @@
+package org.apache.rocketmq.streams.client.windows;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.client.StreamBuilder;
+import org.apache.rocketmq.streams.client.transform.DataStream;
+import org.apache.rocketmq.streams.common.utils.FileUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.batchloader.BatchRowLoader;
+import org.apache.rocketmq.streams.db.driver.batchloader.IRowOperator;
+import org.apache.rocketmq.streams.db.driver.orm.ORMUtil;
+import org.apache.rocketmq.streams.window.model.WindowInstance;
+import org.apache.rocketmq.streams.window.operator.impl.WindowOperator;
+import org.apache.rocketmq.streams.window.state.impl.WindowValue;
+import org.apache.rocketmq.streams.window.storage.WindowStorage;
+import org.junit.Test;
+
+public class WindowHighAvailabilityTest extends AbstractWindowFireModeTest {
+ protected DataStream createSourceDataStream(){
+ return StreamBuilder.dataStream("namespace", "name1")
+ .fromFile("/Users/yuanxiaodong/chris/sls_100000.txt",false);
+ }
+
+ /**
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWindowFireMode0AndNoTimeField() throws InterruptedException {
+ super.testWindowFireMode0(false,60*60);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode1() throws InterruptedException {
+ super.testWindowFireMode1(false,60*5);
+ }
+
+
+
+ @Test
+ public void testWindowFireMode2() {
+ super.testWindowFireMode2(false);
+ }
+
+ @Test
+ public void testLoadData() throws InterruptedException {
+ Map<String,Integer> map=create();
+ WindowOperator windowOperator=new WindowOperator();
+ windowOperator.setNameSpace("namespace");
+ windowOperator.setConfigureName("name1_window_10001");
+ WindowInstance windowInstance=windowOperator.createWindowInstance("2021-07-14 02:00:00","2021-07-14 03:00:00","2021-07-14 03:00:00","1");
+
+ WindowStorage storage= new WindowStorage();
+ System.out.println("1;namespace;name1_window_10001;name1_window_10001;2021-07-14 02:00:00;2021-07-14 03:00:00");
+ System.out.println(windowInstance.createWindowInstanceId());
+ System.out.println(StringUtil.createMD5Str(windowInstance.createWindowInstanceId()));
+ Map<String, WindowValue> windowValueMap=new HashMap<>();
+ AtomicInteger sum=new AtomicInteger(0);
+ BatchRowLoader batchRowLoader = new BatchRowLoader("partition_num",
+ "select * from " + ORMUtil.getTableName(WindowValue.class) + " where window_instance_partition_id ='"
+ + StringUtil.createMD5Str(windowInstance.createWindowInstanceId() )+ "'", new IRowOperator(){
+
+ @Override
+ public void doProcess(Map<String, Object> row) {
+ WindowValue windowValue = ORMUtil.convert(row, WindowValue.class);
+
+ String groupBy=windowValue.getGroupBy();
+ windowValueMap.put(groupBy,windowValue);
+ if(!map.containsKey(groupBy)){
+
+ System.out.println(groupBy+" ======");
+ }else {
+ Integer count=(Integer)windowValue.getComputedColumnResultByKey("total");
+ int mapCount=map.get(groupBy);
+ if(count!=mapCount){
+ System.out.println(groupBy+" ======");
+ }
+ //System.out.println(groupBy+" "+count);
+ sum.addAndGet(count);
+
+ }
+ }
+ });
+ batchRowLoader.startLoadData();
+
+ System.out.println("sum "+sum.get());
+ for(String key:map.keySet()){
+ if(windowValueMap.containsKey(key)==false){
+ System.out.println(key+" "+map.get(key));
+ }
+ }
+
+ Thread.sleep(10000000l);
+ }
+
+
+ protected Map<String,Integer> create(){
+ List<String> lines= FileUtil.loadFileLine("/Users/yuanxiaodong/chris/sls_10000.txt");
+ Map<String,Integer> map=new HashMap<>();
+ for(String line:lines){
+ JSONObject msg=JSONObject.parseObject(line);
+ String projectName=msg.getString("ProjectName");
+ String logstore=msg.getString("LogStore");
+
+ String key= MapKeyUtil.createKey(projectName,logstore);
+
+ if (StringUtil.isEmpty(key)) {
+ key="<null>";
+ }
+ Integer count=map.get(key);
+ if(count==null){
+ map.put(key,1);
+ }else {
+ count++;
+ map.put(key,count);
+ }
+ }
+ return map;
+ }
+
+
+
+}
diff --git a/rocketmq-streams-clients/src/test/resources/log4j.xml b/rocketmq-streams-clients/src/test/resources/log4j.xml
new file mode 100755
index 0000000..a1afd37
--- /dev/null
+++ b/rocketmq-streams-clients/src/test/resources/log4j.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
diff --git a/rocketmq-streams-dim/pom.xml b/rocketmq-streams-dim/pom.xml
new file mode 100644
index 0000000..1175370
--- /dev/null
+++ b/rocketmq-streams-dim/pom.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-dim</artifactId>
+ <name>ROCKETMQ STREAMS :: dim</name>
+ <packaging>jar</packaging>
+ <properties>
+ <file_encoding>UTF-8</file_encoding>
+ <project.build.sourceEncoding>${file_encoding}</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-filter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-http</artifactId>
+ </dependency>
+ </dependencies>
+
+
+</project>
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/DimComponent.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/DimComponent.java
new file mode 100644
index 0000000..8602b5d
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/DimComponent.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim;
+
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.dim.service.IDimService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.dim.service.impl.DimServiceImpl;
+
+public class DimComponent extends AbstractComponent<IDimService> {
+
+ private static final Log LOG = LogFactory.getLog(DimComponent.class);
+
+ // private transient Map<String, DBDim> nameListMap = new HashMap<>();
+
+ protected transient ConfigurableComponent configurableComponent;
+ private transient IDimService dimService;
+
+ public static DimComponent getInstance(String namespace) {
+ return ComponentCreator.getComponent(namespace, ComponentCreator.class);
+ }
+
+ @Override
+ protected boolean startComponent(String namespace) {
+ configurableComponent = ComponentCreator.getComponent(namespace, ConfigurableComponent.class);
+ dimService = new DimServiceImpl(configurableComponent);
+ return true;
+ }
+
+ @Override
+ protected boolean initProperties(Properties properties) {
+ return true;
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ @Override
+ public IDimService getService() {
+ return dimService;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/builder/DimBuilder.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/builder/DimBuilder.java
new file mode 100644
index 0000000..508bb19
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/builder/DimBuilder.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.builder;
+
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+
+public class DimBuilder {
+
+ private String url;
+ private String password;
+ private String userName;
+ protected Long pollingTime = 60L; // 同步数据的事件间隔
+ private String jdbcDriver = "com.mysql.jdbc.Driver";
+
+ public DimBuilder(String url, String userName, String password) {
+ this.url = url;
+ this.password = password;
+ this.userName = userName;
+ }
+
+ public DBDim createDim(String namespace, String name, String sqlOrTableName) {
+ DBDim nameList = new DBDim();
+ nameList.setNameSpace(namespace);
+ if (StringUtil.isNotEmpty(name)) {
+ nameList.setConfigureName(name);
+ }
+ String sql = sqlOrTableName;
+ if (sqlOrTableName.split(" ").length == 1) {
+ sql = "select * from " + sqlOrTableName + " limit 500000";
+ }
+ nameList.setSql(sql);
+ nameList.setJdbcdriver(jdbcDriver);
+ nameList.setPollingTimeMintue(pollingTime);
+ nameList.setUrl(url);
+ nameList.setUserName(userName);
+ nameList.setPassword(password);
+ return nameList;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public Long getPollingTime() {
+ return pollingTime;
+ }
+
+ public void setPollingTime(Long pollingTime) {
+ this.pollingTime = pollingTime;
+ }
+
+ public String getJdbcDriver() {
+ return jdbcDriver;
+ }
+
+ public void setJdbcDriver(String jdbcDriver) {
+ this.jdbcDriver = jdbcDriver;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/InExpressionResource.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/InExpressionResource.java
new file mode 100644
index 0000000..c8a13a7
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/InExpressionResource.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.expression;
+
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.apache.rocketmq.streams.filter.context.RuleContext;
+import org.apache.rocketmq.streams.filter.function.expression.AbstractExpressionFunction;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethodAilas;
+
+import java.util.Map;
+
+@Function
+public class InExpressionResource extends AbstractExpressionFunction {
+
+ /**
+ * value格式 :resourceName.fieldName。如果只有单列，可以省略.fieldname
+ *
+ * @param expression
+ * @param context
+ * @param rule
+ * @return
+ */
+ @FunctionMethod(value = "in_expression_resouce", alias = "in_resouce")
+ @FunctionMethodAilas("in_expression_resouce(resourceName->(varName,functionName,value)&((varName,functionName,"
+ + "value)|(varName,functionName,value)))")
+ @Override
+ public Boolean doExpressionFunction(Expression expression, RuleContext context, Rule rule) {
+ return match(expression, context, rule, false);
+ }
+
+ protected Boolean match(Expression expression, RuleContext context, Rule rule, boolean supportRegex) {
+ Object value = expression.getValue();
+ if (value == null) {
+ return null;
+ }
+ String valueStr = String.valueOf(value);
+ String[] valueArray = valueStr.split("->");
+ String dataResourceNamespace = rule.getNameSpace();
+ String dataResourceName = null;
+ String expressionStr = null;
+ if (valueArray.length == 2) {
+ dataResourceName = valueArray[0];
+ expressionStr = valueArray[1];
+ }
+ if (valueArray.length > 2) {
+ dataResourceNamespace = valueArray[0];
+ dataResourceName = valueArray[1];
+ expressionStr = valueArray[2];
+ }
+
+ DBDim dataResource =
+ (DBDim)context.getConfigurableService().queryConfigurableByIdent(DBDim.TYPE, dataResourceName);
+ if (dataResource == null) {
+ return null;
+ }
+ Map<String, Object> row = dataResource.matchExpression(expressionStr, context.getMessage().getMessageBody());
+ if (row != null && row.size() > 0) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/NotInExpressionResource.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/NotInExpressionResource.java
new file mode 100644
index 0000000..752276c
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/expression/NotInExpressionResource.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.expression;
+
+import org.apache.rocketmq.streams.filter.context.RuleContext;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethodAilas;
+
+@Function
+public class NotInExpressionResource extends InExpressionResource {
+
+ /**
+ * value格式 :resourceName.fieldName。如果只有单列，可以省略.fieldname
+ *
+ * @param expression
+ * @param context
+ * @param rule
+ * @return
+ */
+ @FunctionMethod("not_in_expression_resouce")
+ @FunctionMethodAilas("not_in_expression_resouce(resourceName->(varName,functionName,value)&((varName,"
+ + "functionName,value)|(varName,functionName,value)))")
+ @Override
+ public Boolean doExpressionFunction(Expression expression, RuleContext context, Rule rule) {
+ return !match(expression, context, rule, false);
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceFunction.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceFunction.java
new file mode 100644
index 0000000..c2ffb88
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceFunction.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.script;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.dim.DimComponent;
+import org.apache.rocketmq.streams.dim.intelligence.AbstractIntelligenceCache;
+import org.apache.rocketmq.streams.script.context.FunctionContext;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.script.utils.FunctionUtils;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+@Function
+public class IntelligenceFunction {
+
+ private static final Log LOG = LogFactory.getLog(IntelligenceFunction.class);
+
+ private DimComponent nameListComponent;
+
+ @FunctionMethod(value = "intelligence", alias = "qingbao")
+ public void intelligence(IMessage message, FunctionContext context, String namespace, String nameListName, String intelligenceFieldName, String asName) {
+ intelligenceInner(message, context, namespace, nameListName, intelligenceFieldName, asName, true);
+ }
+
+ @FunctionMethod(value = "left_join_intelligence", alias = "left_join_qingbao")
+ public void intelligenceLeftJoin(IMessage message, FunctionContext context, String namespace, String nameListName, String intelligenceFieldName, String asName) {
+ intelligenceInner(message, context, namespace, nameListName, intelligenceFieldName, asName, false);
+ }
+
+ public void intelligenceInner(IMessage message, FunctionContext context, String namespace, String nameListName, String intelligenceFieldName, String asName, boolean isInner) {
+ String key = FunctionUtils.getValueString(message, context, intelligenceFieldName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ ConfigurableComponent configurableComponent = ComponentCreator.getComponent(namespace, ConfigurableComponent.class);
+ AbstractIntelligenceCache intelligenceCache = configurableComponent.queryConfigurable(AbstractIntelligenceCache.TYPE, nameListName);
+ if (intelligenceCache == null) {
+ throw new RuntimeException("can not query intelligence. the namespace is " + namespace + ", the name is " + nameListName);
+ }
+ Map<String, Object> row = intelligenceCache.getRow(key);
+ if (row != null) {
+ asName = FunctionUtils.getValueString(message, context, asName);
+ if (StringUtil.isNotEmpty(asName)) {
+ asName = asName + ".";
+ } else {
+ asName = "";
+ }
+ for (Entry<String, Object> entry : row.entrySet()) {
+ String elementKey = asName + entry.getKey();
+ message.getMessageBody().put(elementKey, entry.getValue());
+ }
+ } else {
+ if (isInner) {
+ context.breakExecute();
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceNameListFunction.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceNameListFunction.java
new file mode 100644
index 0000000..02d67fb
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/IntelligenceNameListFunction.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.script;
+
+import org.apache.rocketmq.streams.script.annotation.Function;
+
+@Function
+public class IntelligenceNameListFunction {
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/NameListFunction.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/NameListFunction.java
new file mode 100644
index 0000000..3b03a4e
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/function/script/NameListFunction.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.function.script;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.dim.DimComponent;
+import org.apache.rocketmq.streams.script.context.FunctionContext;
+import org.apache.rocketmq.streams.script.annotation.Function;
+import org.apache.rocketmq.streams.script.annotation.FunctionMethod;
+import org.apache.rocketmq.streams.common.context.IMessage;
+import org.apache.rocketmq.streams.script.utils.FunctionUtils;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+@Function
+public class NameListFunction {
+
+ private static final Log LOG = LogFactory.getLog(NameListFunction.class);
+
+ private DimComponent nameListComponent;
+
+ @FunctionMethod(value = "innerJoin", alias = "inner_join")
+ public JSONArray innerJoin(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String alias, String script, String... fieldNames) {
+ JSONArray rows = getRows(message, context, namespace, nameListName, expressionStr, alias, script, fieldNames);
+ if (rows == null || rows.size() == 0) {
+ context.breakExecute();
+ return null;
+ }
+ return rows;
+ }
+
+ @FunctionMethod(value = "leftJoin", alias = "left_join")
+ public JSONArray leftJoin(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String alias, String script, String... fieldNames) {
+ JSONArray rows = getRows(message, context, namespace, nameListName, expressionStr, alias, script, fieldNames);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ return rows;
+ }
+
+ @FunctionMethod(value = "mark_rows", alias = "namelist_rows")
+ public String namelist(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String... fieldNames) {
+ JSONArray rows = getRows(message, context, namespace, nameListName, expressionStr, null, null, fieldNames);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ return rows.toJSONString();
+ }
+
+ @FunctionMethod(value = "mark", alias = "namelist")
+ public String namelist(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String fieldName) {
+ String tmp = fieldName;
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ expressionStr = FunctionUtils.getValueString(message, context, expressionStr);
+ fieldName = FunctionUtils.getValueString(message, context, fieldName);
+ if (fieldName == null) {
+ fieldName = tmp;
+ }
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ Map<String, Object> row =
+ nameListComponent.getService().match(nameListName, expressionStr, message.getMessageBody());
+ if (row != null) {
+ Object value = row.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ return value.toString();
+ }
+ return null;
+ }
+
+ @FunctionMethod(value = "mark", alias = "namelist")
+ public String namelist(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr, String fieldNames, String joinMark) {
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ expressionStr = FunctionUtils.getValueString(message, context, expressionStr);
+ fieldNames = FunctionUtils.getValueString(message, context, fieldNames);
+ joinMark = FunctionUtils.getValueString(message, context, joinMark);
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ Map<String, Object> row =
+ nameListComponent.getService().match(nameListName, expressionStr, message.getMessageBody());
+ if (row != null) {
+ String[] fieldNameTem = fieldNames.split(",");
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < fieldNameTem.length; i++) {
+ Object tem = row.get(fieldNameTem[i]);
+ if (tem != null) {
+ if (result.length() == 0) {
+ if (StringUtil.isNotEmpty(tem.toString()) && !("null".equalsIgnoreCase(tem.toString()))) {
+ result.append(tem);
+ }
+ } else {
+ if (StringUtil.isNotEmpty(tem.toString()) && !("null".equalsIgnoreCase(tem.toString()))) {
+ result.append(joinMark + tem);
+ }
+ }
+ }
+
+ }
+ return result.toString();
+ }
+ return null;
+ }
+
+ @FunctionMethod(value = "in_namelist", alias = "in_namelist")
+ public Boolean inNameList(IMessage message, FunctionContext context, String namespace, String nameListName,
+ String expressionStr) {
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ Map<String, Object> row =
+ nameListComponent.getService().match(nameListName, expressionStr, message.getMessageBody());
+ if (row != null && row.size() > 0) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * 根据表达式，从namelist中获取符合条件的数据
+ *
+ * @param message
+ * @param context
+ * @param namespace
+ * @param nameListName
+ * @param expressionStr （varname,functionName,value)&(varname,functionName,value)
+ * @param fieldNames 需要返回的字段名
+ * @return
+ */
+ protected JSONArray getRows(IMessage message, FunctionContext context, String namespace, String nameListName, String expressionStr, String alias, String script, String... fieldNames) {
+ nameListName = FunctionUtils.getValueString(message, context, nameListName);
+ namespace = FunctionUtils.getValueString(message, context, namespace);
+ expressionStr = FunctionUtils.getValueString(message, context, expressionStr);
+ script = FunctionUtils.getValueString(message, context, script);
+ if (StringUtil.isEmpty(script)) {
+ script = null;
+ }
+ nameListComponent = ComponentCreator.getComponent(namespace, DimComponent.class);
+ List<Map<String, Object>> rows =
+ nameListComponent.getService().matchSupportMultiRow(nameListName, expressionStr, message.getMessageBody(), script);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ JSONArray jsonArray = new JSONArray();
+ for (Map<String, Object> row : rows) {
+ JSONObject jsonObject = new JSONObject();
+ if (fieldNames == null || fieldNames.length == 0) {
+ if (alias == null) {
+ jsonObject.putAll(row);
+ } else {
+ Iterator<Entry<String, Object>> it = row.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<String, Object> entry = it.next();
+ String fieldName = entry.getKey();
+ if (alias != null) {
+ fieldName = alias + "." + fieldName;
+ }
+ jsonObject.put(fieldName, entry.getValue());
+ }
+ }
+
+ } else {
+ for (String fieldName : fieldNames) {
+ String tmp = fieldName;
+ if (alias != null) {
+ fieldName = alias + "." + fieldName;
+ }
+ jsonObject.put(fieldName, row.get(tmp));
+ }
+ }
+ jsonArray.add(jsonObject);
+ }
+ return jsonArray;
+ }
+
+}
\ No newline at end of file
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/DimIndex.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/DimIndex.java
new file mode 100644
index 0000000..6c6aeef
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/DimIndex.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.index;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.datatype.IntDataType;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.rocketmq.streams.common.cache.CompressTable;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DimIndex {
+
+ private static final Log LOG = LogFactory.getLog(DimIndex.class);
+
+ /**
+ * 索引字段名，支持多个索引，每个索引一行，支持组合索引，多个字段用；拼接 name 单索引 name;age 组合索引
+ */
+ protected List<String> indexs = new ArrayList<>();
+
+ /**
+ * 如果是唯一索引，用压缩值存储 每个索引一行
+ */
+ protected Map<String, IntValueKV> uniqueIndex;
+
+ /**
+ * 如果是非唯一索引，用这个结构存储 每个索引一行，后面的map：key：索引值；value：row id 列表，rowid用字节表示
+ */
+ protected Map<String, Map<String, List<Integer>>> mutilIndex = new HashMap<>();
+
+ protected boolean isUnique = false;//如果是唯一索引，值为true
+
+ public DimIndex(List<String> indexs) {
+ this.indexs = formatIndexs(indexs);
+ }
+
+ public DimIndex(String index, String... indexs) {
+ if (indexs == null) {
+ return;
+ }
+ List<String> indexList = new ArrayList<>();
+ for (String idx : indexs) {
+ indexList.add(idx);
+ }
+ this.indexs = formatIndexs(indexList);
+ }
+
+ /**
+ * 组合索引，多个字段要以名称顺序排列，完成索引名称的标注化处理
+ *
+ * @param indexs
+ * @return
+ */
+ protected List<String> formatIndexs(List<String> indexs) {
+ List<String> allIndex = new ArrayList<>();
+ for (String indexName : indexs) {
+ String[] values = indexName.split(";");
+ List<String> indexList = new ArrayList<>();
+ for (String value : values) {
+ indexList.add(value);
+ }
+ Collections.sort(indexList);
+ String indexKey = MapKeyUtil.createKey(indexList);
+ allIndex.add(indexKey);
+ }
+ return allIndex;
+ }
+
+ /**
+ * 加载一行数据，如果是唯一索引，在uniqueIndex中查找，否则在mutilIndex查找
+ *
+ * @param indexName 索引名，如name
+ * @param indexValue 索引值，如chris
+ * @return
+ */
+ public List<Integer> getRowIds(String indexName, String indexValue) {
+ if (isUnique) {
+ IntValueKV index = this.uniqueIndex.get(indexName);
+ if (index == null) {
+ return null;
+ }
+ Integer rowId = index.get(indexValue);
+ if (rowId == null) {
+ return null;
+ }
+ List<Integer> rowIds = new ArrayList<>();
+ rowIds.add(rowId);
+ return rowIds;
+ } else {
+ Map<String, List<Integer>> indexs = this.mutilIndex.get(indexName);
+ if (indexs == null) {
+ return null;
+ }
+ return indexs.get(indexValue);
+ }
+ }
+
+ /**
+ * 构建索引，如果是唯一索引，构建在uniqueIndex数据结构中，否则构建在mutilIndex这个数据结构中
+ *
+ * @param tableCompress 表数据
+ */
+ public void buildIndex(CompressTable tableCompress) {
+
+ if (isUnique) {
+ Map<String, IntValueKV> fieldIndex2RowIndex = new HashMap<>();
+ buildUniqueIndex(tableCompress, fieldIndex2RowIndex);
+ this.uniqueIndex = fieldIndex2RowIndex;
+ } else {
+ Map<String, Map<String, List<Integer>>> fieldIndex2RowIndex = new HashMap<>();
+ buildIndex(tableCompress, fieldIndex2RowIndex);
+ this.mutilIndex = fieldIndex2RowIndex;
+ }
+ }
+
+ /**
+ * 构建唯一索引
+ *
+ * @param tableCompress
+ * @param fieldIndex2RowIndex
+ */
+ protected void buildUniqueIndex(CompressTable tableCompress, Map<String, IntValueKV> fieldIndex2RowIndex) {
+ if (indexs == null || indexs.size() == 0) {
+ return;
+ }
+
+ int i = 0;
+ Iterator<Map<String, Object>> it = tableCompress.newIterator();
+ while (it.hasNext()) {
+ /**
+ * 为每个索引做构建
+ */
+ for (String indexName : indexs) {
+ IntValueKV index = fieldIndex2RowIndex.get(indexName);
+ if (index == null) {
+ synchronized (this) {
+ index = fieldIndex2RowIndex.get(indexName);
+ if (index == null) {
+ index = new IntValueKV(tableCompress.getRowCount());
+ fieldIndex2RowIndex.put(indexName, index);
+ }
+ }
+
+ }
+ String[] nameIndexs = indexName.split(";");
+ Arrays.sort(nameIndexs);
+ Map<String, String> cacheValues = createRow(it.next());
+ String indexValue = createIndexValue(cacheValues, nameIndexs);
+ index.put(indexValue, i);
+ }
+ i++;
+ }
+ LOG.info(" finish poll data , the row count is " + i + ". byte is " + tableCompress
+ .getByteCount());
+ }
+
+ /**
+ * 从table compress 中取出所有的行，构建索引。把
+ *
+ * @param dataCacheVar
+ * @param fieldIndex2RowIndex
+ */
+ protected void buildIndex(CompressTable dataCacheVar, Map<String, Map<String, List<Integer>>> fieldIndex2RowIndex) {
+ Iterator<Map<String, Object>> it = dataCacheVar.newIterator();
+ int i = 0;
+ while (it.hasNext()) {
+ Map<String, Object> row = it.next();
+ Map<String, String> cacheValues = createRow(row);
+ putValue2Index(cacheValues, i, fieldIndex2RowIndex);
+ i++;
+ }
+
+ LOG.info(" finish poll data , the row count is " + i + ". byte is " + dataCacheVar
+ .getByteCount());
+
+ }
+
+ /**
+ * 创建索引结构
+ *
+ * @param value
+ * @param rowIndex
+ * @param fieldIndex2RowIndex
+ */
+ protected void putValue2Index(Map<String, String> value, Integer rowIndex,
+ Map<String, Map<String, List<Integer>>> fieldIndex2RowIndex) {
+ if (indexs == null || indexs.size() == 0) {
+ return;
+ }
+ for (String indexName : indexs) {
+ Map<String, List<Integer>> name2RowIndexs = fieldIndex2RowIndex.get(indexName);
+ if (name2RowIndexs == null) {
+ synchronized (this) {
+ name2RowIndexs = fieldIndex2RowIndex.get(indexName);
+ if (name2RowIndexs == null) {
+ name2RowIndexs = new HashMap<>();
+ fieldIndex2RowIndex.put(indexName, name2RowIndexs);
+ }
+ }
+
+ }
+ String[] nameIndexs = indexName.split(";");
+ Arrays.sort(nameIndexs);
+ addValue2Index(indexName, name2RowIndexs, rowIndex, value, nameIndexs);
+ }
+ }
+
+ /**
+ * 根据索引名称，把不同的索引值创建key，value放入索引缓存
+ *
+ * @param indexName 索引名称，多字段索引用；分割
+ * @param name2RowIndexs 索引值，list<id>
+ * @param rowIndex 行号
+ * @param row
+ */
+ protected void addValue2Index(String indexName, Map<String, List<Integer>> name2RowIndexs, Integer rowIndex,
+ Map<String, String> row, String[] nameIndexs) {
+ String indexValue = createIndexValue(row, nameIndexs);
+ addValue2Index(name2RowIndexs, rowIndex, indexValue);
+ }
+
+ /**
+ * 对于组合索引，把各个字段的值取出来
+ *
+ * @param row
+ * @param nameIndexs
+ * @return
+ */
+ protected String createIndexValue(Map<String, String> row, String[] nameIndexs) {
+ String[] indexValues = new String[nameIndexs.length];
+ for (int i = 0; i < nameIndexs.length; i++) {
+ indexValues[i] = row.get(nameIndexs[i]);
+ }
+ if (indexValues != null && indexValues.length > 0) {
+ String indexValue = MapKeyUtil.createKey(indexValues);
+ return indexValue;
+ }
+ return null;
+ }
+
+ /**
+ * 把row 中非string的值转化成string
+ *
+ * @param row
+ * @return
+ */
+ protected Map<String, String> createRow(Map<String, Object> row) {
+ Map<String, String> cacheValues = new HashMap<String, String>();//一行数据
+ Iterator<Map.Entry<String, Object>> iterator = row.entrySet().iterator();
+ //把数据value从object转化成string
+ while (iterator.hasNext()) {
+ Map.Entry<String, Object> entry = iterator.next();
+ if (entry != null && entry.getValue() != null && entry.getKey() != null) {
+ cacheValues.put(entry.getKey(), entry.getValue().toString());
+ }
+ }
+ return cacheValues;
+ }
+
+ public static IntDataType INTDATATYPE = new IntDataType();
+
+ /**
+ * 把确定多索引值，和行号放入索引缓存
+ *
+ * @param name2RowIndexs
+ * @param rowIndex
+ * @param indexValue
+ */
+ protected void addValue2Index(Map<String, List<Integer>> name2RowIndexs, Integer rowIndex,
+ String indexValue) {
+
+ List<Integer> rowIndexs = name2RowIndexs.get(indexValue);
+ if (rowIndexs == null) {
+ rowIndexs = new ArrayList<>();
+ name2RowIndexs.put(indexValue, rowIndexs);
+ }
+ rowIndexs.add(rowIndex);
+
+ }
+
+ public boolean isUnique() {
+ return isUnique;
+ }
+
+ public void setUnique(boolean unique) {
+ isUnique = unique;
+ }
+
+ public List<String> getIndexs() {
+ return indexs;
+ }
+
+ public void setIndexs(List<String> indexs) {
+ this.indexs = indexs;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/IndexExecutor.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/IndexExecutor.java
new file mode 100644
index 0000000..ceb4f52
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/index/IndexExecutor.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.index;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.function.expression.Equals;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression;
+import org.apache.rocketmq.streams.script.ScriptComponent;
+import org.apache.rocketmq.streams.common.datatype.IntDataType;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+
+/**
+ * 执行索引的查询和构建。主要是完成表达式的解析，对于等值的表达式字段，如果有索引，根据索引查询，然后执行非等值部分的判断
+ */
+public class IndexExecutor {
+
+ private String expressionStr;//表达式
+
+ private boolean isSupport = false;//是否支持索引，比如表达式等值部分无所以，则不能走索引逻辑
+
+ private String namespace;
+
+ private String indexNameKey;//索引的名字，多个字段";"分隔
+
+ private List<String> msgNames;//
+
+ private Rule rule;//表达式会被编译成rule
+
+ private List<String> index; //标准化后的索引name
+
+ private Set<String> indexNames = new HashSet<>();
+
+ public IndexExecutor(String expressionStr, String namespace, List<String> index) {
+ this.expressionStr = expressionStr;
+ this.namespace = namespace;
+
+ List<String> allIndex = new ArrayList<>();
+ for (String indexName : index) {
+ String[] values = indexName.split(";");
+ List<String> indexList = new ArrayList<>();
+ for (String value : values) {
+ indexNames.add(value);
+ indexList.add(value);
+ }
+ Collections.sort(indexList);
+ String indexKey = MapKeyUtil.createKey(indexList);
+ allIndex.add(indexKey);
+ }
+ this.index = allIndex;
+ parse();
+ }
+
+ /**
+ * 解析表达式，找出等值字段和非等值字段 如果有索引走索引，否则走全表扫描
+ */
+ protected void parse() {
+ List<Expression> expressions = new ArrayList<>();
+ List<RelationExpression> relationExpressions = new ArrayList<>();
+ Expression expression = ExpressionBuilder.createOptimizationExpression(namespace, "tmp", expressionStr,
+ expressions, relationExpressions);
+
+ RelationExpression relationExpression = null;
+ if (RelationExpression.class.isInstance(expression)) {
+ relationExpression = (RelationExpression)expression;
+ if (!"and".equals(relationExpression.getRelation())) {
+ isSupport = false;
+ return;
+ }
+ }
+
+ this.isSupport = true;
+ List<Expression> indexExpressions = new ArrayList<>();
+ List<Expression> otherExpressions = new ArrayList<>();
+ if (relationExpression != null) {
+ Map<String, Expression> map = new HashMap<>();
+ for (Expression tmp : expressions) {
+ map.put(tmp.getConfigureName(), tmp);
+ }
+ for (Expression tmp : relationExpressions) {
+ map.put(tmp.getConfigureName(), tmp);
+ }
+ List<String> expressionNames = relationExpression.getValue();
+ relationExpression.setValue(new ArrayList<>());
+ for (String expressionName : expressionNames) {
+ Expression subExpression = map.get(expressionName);
+ if (subExpression != null && !RelationExpression.class.isInstance(subExpression)) {
+ indexExpressions.add(subExpression);
+ } else {
+ otherExpressions.add(subExpression);
+ relationExpression.getValue().add(subExpression.getConfigureName());
+ }
+ }
+
+ } else {
+ indexExpressions.add(expression);
+ }
+
+ List<String> fieldNames = new ArrayList<>();
+ Map<String, String> msgNames = new HashMap<>();
+
+ for (Expression expre : indexExpressions) {
+ if (RelationExpression.class.isInstance(expre)) {
+ continue;
+ }
+ String indexName = expre.getValue().toString();
+ if (Equals.isEqualFunction(expre.getFunctionName()) && indexNames.contains(indexName)) {
+
+ fieldNames.add(indexName);
+ msgNames.put(indexName, expre.getVarName());
+ }
+ }
+ Collections.sort(fieldNames);
+ indexNameKey = MapKeyUtil.createKey(fieldNames);
+ if (!this.index.contains(indexNameKey)) {
+ this.isSupport = false;
+ return;
+ }
+ this.msgNames = createMsgNames(fieldNames, msgNames);
+ if (otherExpressions.size() == 0) {
+ return;
+ }
+ Rule rule = null;
+ if (relationExpression == null) {
+ rule = ExpressionBuilder.createRule(namespace, "tmpRule", expression);
+ } else {
+ rule = ExpressionBuilder.createRule(namespace, "tmpRule", expression, expressions, relationExpressions);
+ }
+
+ this.rule = rule;
+
+ }
+
+ /**
+ * 创建索引字段的索引值
+ *
+ * @param fieldNames
+ * @param msgNames
+ * @return
+ */
+ protected List<String> createMsgNames(List<String> fieldNames, Map<String, String> msgNames) {
+ List<String> msgNameList = new ArrayList<>();
+ for (String fieldName : fieldNames) {
+ msgNameList.add(msgNames.get(fieldName));
+ }
+ return msgNameList;
+ }
+
+ public boolean isSupport() {
+ return isSupport;
+ }
+
+ private static IntDataType intDataType = new IntDataType();
+
+ public List<Map<String, Object>> match(JSONObject msg, AbstractDim nameList, boolean needAll) {
+ return match(msg, nameList, needAll);
+ }
+
+ public List<Map<String, Object>> match(JSONObject msg, AbstractDim nameList, boolean needAll, String script) {
+ List<Map<String, Object>> rows = new ArrayList<>();
+ String msgValue = createValue(msg);
+ List<Integer> rowIds = nameList.findRowIdByIndex(indexNameKey, msgValue);
+ if (rowIds == null) {
+ return null;
+ }
+ for (Integer rowId : rowIds) {
+ Map<String, Object> oldRow = nameList.getDataCache().getRow(rowId);
+ Map<String, Object> newRow = executeScript(oldRow, script);
+ if (rule == null) {
+ rows.add(newRow);
+ if (needAll == false) {
+ return rows;
+ }
+ continue;
+ }
+ Rule ruleTemplete = this.rule;
+ Rule rule = ruleTemplete.copy();
+ Map<String, Expression> expressionMap = new HashMap<>();
+ for (Expression expression : rule.getExpressionMap().values()) {
+ expressionMap.put(expression.getConfigureName(), expression);
+ if (RelationExpression.class.isInstance(expression)) {
+ continue;
+ }
+ Object object = expression.getValue();
+ if (object != null && DataTypeUtil.isString(object.getClass())) {
+ String fieldName = (String)object;
+ Object o = newRow.get(fieldName);
+ if (o != null) {
+ Expression e = expression.copy();
+ e.setValue(o.toString());
+ expressionMap.put(e.getConfigureName(), e);
+ }
+ }
+ }
+ rule.setExpressionMap(expressionMap);
+ boolean matched = rule.execute(msg);
+ if (matched) {
+ rows.add(newRow);
+ if (needAll == false) {
+ return rows;
+ }
+ }
+ }
+ return rows;
+ }
+
+ protected Map<String, Object> executeScript(Map<String, Object> oldRow, String script) {
+ if (script == null) {
+ return oldRow;
+ }
+ ScriptComponent scriptComponent = ScriptComponent.getInstance();
+ JSONObject msg = new JSONObject();
+ msg.putAll(oldRow);
+ scriptComponent.getService().executeScript(msg, script);
+ return msg;
+ }
+
+ /**
+ * 按顺序创建msg的key
+ *
+ * @param msg
+ * @return
+ */
+ private String createValue(JSONObject msg) {
+ List<String> value = new ArrayList<>();
+ for (String msgName : msgNames) {
+ value.add(msg.getString(msgName));
+ }
+ return MapKeyUtil.createKey(value);
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AbstractIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AbstractIntelligenceCache.java
new file mode 100644
index 0000000..65b27f7
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AbstractIntelligenceCache.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.http.source.util.HttpUtil;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.configurable.IConfigurableService;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.rocketmq.streams.common.channel.sink.ISink;
+import org.apache.rocketmq.streams.common.utils.NumberUtils;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.dboperator.IDBDriver;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+
+public abstract class AbstractIntelligenceCache extends BasedConfigurable implements
+ IAfterConfiguableRefreshListerner {
+
+ private static final Log LOG = LogFactory.getLog(AbstractIntelligenceCache.class);
+
+ public static final String TYPE = "intelligence";
+
+ protected static final int FILE_MAX_LINE = 50000;//每个文件最大行数
+
+ protected transient IntValueKV intValueKV = new IntValueKV(0) {
+ @Override
+ public Integer get(String key) {
+ return null;
+ }
+
+ @Override
+ public void put(String key, Integer value) {
+
+ }
+ };
+
+ protected String idFieldName = "id";//必须有id字段
+
+ protected int batchSize = 3000;
+
+ @ENVDependence
+ protected Long pollingTimeMintue = 30L;
+
+ protected String datasourceName;//情报对应的存储
+
+ protected transient IDBDriver outputDataSource;
+
+ protected static ExecutorService executorService;
+
+ protected transient ScheduledExecutorService scheduledExecutorService;
+
+ public AbstractIntelligenceCache() {
+ setType(TYPE);
+ executorService = new ThreadPoolExecutor(20, 20,
+ 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(1000));
+ scheduledExecutorService = new ScheduledThreadPoolExecutor(3);
+ }
+
+ public IntValueKV startLoadData(String sql, IDBDriver resource) {
+ try {
+ String statisticalSQL = sql;
+ int startIndex = sql.toLowerCase().indexOf("from");
+ statisticalSQL = "select count(1) as c, min(" + idFieldName + ") as min, max(" + idFieldName + ") as max "
+ + sql.substring(startIndex);
+ List<Map<String, Object>> rows = resource.queryForList(statisticalSQL);
+ Map<String, Object> row = rows.get(0);
+ int count = Integer.valueOf(row.get("c").toString());
+ IntValueKV intValueKV = new IntValueKV(count);
+ //int maxBatch=count/maxSyncCount;//每1w条数据，一个并发。如果数据量比较大，为了提高性能，并行执行
+ if (count == 0) {
+ return new IntValueKV(0) {
+ @Override
+ public Integer get(String key) {
+ return null;
+ }
+
+ @Override
+ public void put(String key, Integer value) {
+
+ }
+ };
+ }
+ long min = Long.valueOf(row.get("min").toString());
+ long max = Long.valueOf(row.get("max").toString());
+ int maxSyncCount = count / FILE_MAX_LINE + 1;
+ long step = (max - min + 1) / maxSyncCount;
+ CountDownLatch countDownLatch = new CountDownLatch(maxSyncCount + 1);
+ AtomicInteger finishedCount = new AtomicInteger(0);
+ String taskSQL = null;
+ if (sql.indexOf(" where ") != -1) {
+ taskSQL = sql + " and " + idFieldName + ">#{startIndex} and " + idFieldName + "<=#{endIndex} order by "
+ + idFieldName + " limit " + batchSize;
+ } else {
+ taskSQL = sql + " where " + idFieldName + ">#{startIndex} and " + idFieldName
+ + "<=#{endIndex} order by " + idFieldName + " limit " + batchSize;
+ }
+
+ int i = 0;
+ for (; i < maxSyncCount; i++) {
+ FetchDataTask fetchDataTask = new FetchDataTask(taskSQL, (min - 1) + step * i,
+ (min - 1) + step * (i + 1), countDownLatch, finishedCount, resource, i, intValueKV, this, count);
+ executorService.execute(fetchDataTask);
+ }
+ FetchDataTask fetchDataTask = new FetchDataTask(taskSQL, (min - 1) + step * i, (min - 1) + step * (i + 1),
+ countDownLatch, finishedCount, resource, i, intValueKV, this, count);
+ executorService.execute(fetchDataTask);
+
+ countDownLatch.await();
+
+ LOG.info(getClass().getSimpleName() + " load data finish, load data line size is " + intValueKV.getSize());
+ return intValueKV;
+ } catch (Exception e) {
+ LOG.error("failed loading intelligence data!", e);
+ return new IntValueKV(0) {
+ @Override
+ public Integer get(String key) {
+ return null;
+ }
+
+ @Override
+ public void put(String key, Integer value) {
+
+ }
+ };
+ }
+ }
+
+ protected transient AtomicBoolean hasInit = new AtomicBoolean(false);
+
+ @Override
+ public void doProcessAfterRefreshConfigurable(IConfigurableService configurableService) {
+ this.outputDataSource = configurableService.queryConfigurable(ISink.TYPE, datasourceName);
+ }
+
+ public void startIntelligence() {
+ boolean success = dbInit();
+ if (success) {
+ startIntelligenceInner();
+ } else {
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ boolean success = false;
+ while (!success) {
+ success = dbInit();
+ try {
+ Thread.sleep(60 * 1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ startIntelligenceInner();
+ }
+ });
+ thread.start();
+ }
+ }
+
+ public void startIntelligenceInner() {
+ String sql = getSQL();
+ if (hasInit.compareAndSet(false, true)) {
+ this.intValueKV = startLoadData(sql, outputDataSource);
+ scheduledExecutorService.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ intValueKV = startLoadData(sql, outputDataSource);
+ }
+ }, pollingTimeMintue, pollingTimeMintue, TimeUnit.MINUTES);
+ }
+ }
+
+ public abstract Map<String, Object> getRow(String key);
+
+ /**
+ * 查询情报需要的sql
+ *
+ * @return
+ */
+ protected abstract String getSQL();
+
+ /**
+ * 情报中的 情报字段名
+ *
+ * @return
+ */
+ public abstract String getKeyName();
+
+ /**
+ * 情报对应的表名
+ *
+ * @return
+ */
+ public abstract String getTableName();
+
+ protected class FetchDataTask implements Runnable {
+ IntValueKV intValueKV;
+ long startIndex;
+ long endIndex;
+ String sql;
+ CountDownLatch countDownLatch;
+ int index;
+ IDBDriver resource;
+ AtomicInteger finishedCount;//完成了多少条
+ AbstractIntelligenceCache cache;
+ int totalSize;//一共有多少条数据
+
+ public FetchDataTask(String sql, long startIndex, long endIndex, CountDownLatch countDownLatch,
+ AtomicInteger finishedCount, IDBDriver resource, int i, IntValueKV intValueKV, AbstractIntelligenceCache cache, int totalSize) {
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ this.countDownLatch = countDownLatch;
+ this.sql = sql;
+ this.finishedCount = finishedCount;
+ this.resource = resource;
+ this.index = i;
+ this.intValueKV = intValueKV;
+ this.cache = cache;
+ this.totalSize = totalSize;
+ }
+
+ @Override
+ public void run() {
+ long currentIndex = startIndex;
+ JSONObject msg = new JSONObject();
+ msg.put("endIndex", endIndex);
+ while (true) {
+ try {
+
+ msg.put("startIndex", currentIndex);
+
+ String sql = SQLUtil.parseIbatisSQL(msg, this.sql);
+ List<Map<String, Object>> rows = resource.queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ break;
+ }
+ currentIndex = Long.valueOf(rows.get(rows.size() - 1).get(idFieldName).toString());
+
+ int size = rows.size();
+ int count = finishedCount.addAndGet(size);
+ double progress = (double)count / (double)totalSize;
+ progress = progress * 100;
+ System.out.println(cache.getClass().getSimpleName() + ", finished count is " + count + " the total count is " + totalSize + ", the progress is " + String.format("%.2f", progress) + "%");
+ if (size < batchSize) {
+ if (size > 0) {
+ doProccRows(intValueKV, rows, index);
+ }
+ break;
+ }
+ doProccRows(intValueKV, rows, index);
+ } catch (Exception e) {
+ throw new RuntimeException("put data error ", e);
+ }
+ }
+
+ countDownLatch.countDown();
+ }
+ }
+
+ public boolean dbInit() {
+ String uri = "/api/rest/rds/getDataSource";
+ String httpPrefix = "http";
+ String httpProtocol = "http://";
+ try {
+ String dbEndpoint = ComponentCreator.getProperties().getProperty(
+ ConfigureFileKey.INTELLIGENCE_TIP_DB_ENDPOINT);
+ if (StringUtils.isNotBlank(dbEndpoint)) {
+ if (!dbEndpoint.startsWith(httpPrefix)) {
+ dbEndpoint = httpProtocol + dbEndpoint;
+ }
+ String content = HttpUtil.getContent(dbEndpoint + uri);
+ if (StringUtils.isNotBlank(content)) {
+ JSONObject obj = JSON.parseObject(content);
+ String statusCode = "code";
+ int successCode = 200;
+ if (obj.getInteger(statusCode) == successCode) {
+ JSONObject data = obj.getJSONObject("data");
+ JSONObject dbInfo = data.getJSONObject("dBInfo");
+ if (dbInfo != null) {
+ String dbUrl = "jdbc:mysql://" + dbInfo.getString("dbConnection") + ":" + dbInfo.getInteger(
+ "port") + "/" + dbInfo.getString("dBName");
+ String dbUserName = dbInfo.getString("userName");
+ String dbPassword = dbInfo.getString("passWord");
+ JDBCDriver dataSource = (JDBCDriver)this.outputDataSource;
+ dataSource.setUrl(dbUrl);
+ dataSource.setPassword(dbPassword);
+ dataSource.setUserName(dbUserName);
+ dataSource.setHasInit(false);
+ dataSource.init();
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ /**
+ * 把存储0/1字符串的值，转化成bit
+ *
+ * @param values
+ * @return
+ */
+ protected int createInt(List<String> values) {
+ return NumberUtils.createBitMapInt(values);
+ }
+
+ /**
+ * 获取某位的值，如果是1，返回字符串1，否则返回null
+ *
+ * @param num
+ * @param i
+ * @return
+ */
+ protected String getNumBitValue(int num, int i) {
+ boolean exist = NumberUtils.getNumFromBitMapInt(num, i);
+ if (exist) {
+ return "1";
+ }
+ return null;
+ }
+
+ protected abstract void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index);
+
+ public String getIdFieldName() {
+ return idFieldName;
+ }
+
+ public void setIdFieldName(String idFieldName) {
+ this.idFieldName = idFieldName;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public Long getPollingTimeMintue() {
+ return pollingTimeMintue;
+ }
+
+ public void setPollingTimeMintue(Long pollingTimeMintue) {
+ this.pollingTimeMintue = pollingTimeMintue;
+ }
+
+ public String getDatasourceName() {
+ return datasourceName;
+ }
+
+ public void setDatasourceName(String datasourceName) {
+ this.datasourceName = datasourceName;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AccountIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AccountIntelligenceCache.java
new file mode 100644
index 0000000..9466f3e
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/AccountIntelligenceCache.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * table: ads_yunsec_abnormal_account
+ */
+public class AccountIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+
+ private static final Log LOG = LogFactory.getLog(AccountIntelligenceCache.class);
+
+ /**
+ * 情报域名
+ */
+ protected transient String keyName = "account";
+
+ @Override
+ public Map<String, Object> getRow(String account) {
+ Integer value = intValueKV.get(account);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<String, Object>() {{
+ put("account", account);
+ }};
+ return row;
+ }
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id, `account` FROM `ads_yunsec_abnormal_account`";
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_abnormal_account";
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ rows.forEach(row -> {
+ String account = (String)row.get(keyName);
+ if (account != null) {
+ intValueKV.put(account, 1);
+ }
+ });
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/DomainIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/DomainIntelligenceCache.java
new file mode 100644
index 0000000..72a88bf
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/DomainIntelligenceCache.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DomainIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+ private static final Log LOG = LogFactory.getLog(DomainIntelligenceCache.class);
+ protected transient String keyName = "domain";
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id, `domain` , `is_malicious_source` , `is_phishing` , `is_c2` , `is_mining_pool` FROM `ads_yunsec_ti_domain_all_df` where curdate() < date_add(modify_time, interval expire_time day) ";
+ }
+
+ @Override
+ public Map<String, Object> getRow(String ip) {
+ Integer value = intValueKV.get(ip);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<>();
+ row.put("is_malicious_source", getNumBitValue(value, 0));
+ row.put("is_phishing", getNumBitValue(value, 1));
+ row.put("is_c2", getNumBitValue(value, 2));
+ row.put("is_mining_pool", getNumBitValue(value, 3));
+ return row;
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_ti_domain_all_df";
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ for (Map<String, Object> row : rows) {
+ String ip = (String)row.get(keyName);
+ if (ip == null) {
+ LOG.warn("load Intelligence exception ,the ip is null");
+ continue;
+ }
+ List<String> values = new ArrayList<>();
+ values.add((String)row.get("is_malicious_source"));
+ values.add((String)row.get("is_phishing"));
+ values.add((String)row.get("is_c2"));
+ values.add((String)row.get("is_mining_pool"));
+ int value = createInt(values);
+ synchronized (this) {
+ intValueKV.put(ip, value);
+ }
+
+ }
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/IPIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/IPIntelligenceCache.java
new file mode 100644
index 0000000..6382cd2
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/IPIntelligenceCache.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.dboperator.IDBDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+
+public class IPIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+ private static final Log LOG = LogFactory.getLog(IPIntelligenceCache.class);
+ protected transient String keyName = "ip";
+
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id,ip, `is_web_attack` , `is_tor` , `is_proxy` , `is_nat` , `is_mining_pool` , `is_c2` , "
+ + "`is_malicious_source` , `is_3rd` , `is_idc` , `is_malicious_login` FROM `ads_yunsec_ti_ip_all_df` where curdate() < date_add(modify_time, interval expire_time day) ";
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_ti_ip_all_df";
+ }
+
+ @Override
+ public Map<String, Object> getRow(String ip) {
+
+ Integer value = intValueKV.get(ip);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<>();
+
+ row.put("is_web_attack", getNumBitValue(value, 0));
+ row.put("is_tor", getNumBitValue(value, 1));
+ row.put("is_proxy", getNumBitValue(value, 2));
+ row.put("is_nat", getNumBitValue(value, 3));
+ row.put("is_mining_pool", getNumBitValue(value, 4));
+ row.put("is_c2", getNumBitValue(value, 5));
+ row.put("is_malicious_source", getNumBitValue(value, 6));
+ row.put("is_3rd", getNumBitValue(value, 7));
+ row.put("is_idc", getNumBitValue(value, 8));
+ row.put("is_malicious_login", getNumBitValue(value, 9));
+ return row;
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ for (Map<String, Object> row : rows) {
+ String ip = (String)row.get(keyName);
+ if (ip == null) {
+ LOG.warn("load Intelligence exception ,the ip is null");
+ continue;
+ }
+ List<String> values = new ArrayList<>();
+ values.add((String)row.get("is_web_attack"));
+ values.add((String)row.get("is_tor"));
+ values.add((String)row.get("is_proxy"));
+ values.add((String)row.get("is_nat"));
+ values.add((String)row.get("is_mining_pool"));
+ values.add((String)row.get("is_c2"));
+ values.add((String)row.get("is_malicious_source"));
+ values.add((String)row.get("is_3rd"));
+ values.add((String)row.get("is_idc"));
+ values.add((String)row.get("is_malicious_login"));
+ int value = createInt(values);
+ synchronized (this) {
+ intValueKV.put(ip, value);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ ComponentCreator.setProperties(
+ "/Users/yuanxiaodong/Documents/workdir/档案/阿里安全/专有云/2020/dipper-siem/siem.properties");
+ IPIntelligenceCache ipIntelligenceCache = new IPIntelligenceCache();
+ IDBDriver outputDataSource = DriverBuilder.createDriver();
+ ipIntelligenceCache.startLoadData(ipIntelligenceCache.getSQL(), outputDataSource);
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/URLIntelligenceCache.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/URLIntelligenceCache.java
new file mode 100644
index 0000000..8910469
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/intelligence/URLIntelligenceCache.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.intelligence;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rocketmq.streams.common.configurable.IAfterConfiguableRefreshListerner;
+import org.apache.rocketmq.streams.common.cache.compress.impl.IntValueKV;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class URLIntelligenceCache extends AbstractIntelligenceCache implements IAfterConfiguableRefreshListerner {
+
+ private static final Log LOG = LogFactory.getLog(URLIntelligenceCache.class);
+
+ protected transient String keyName = "url";
+
+ @Override
+ protected String getSQL() {
+ return "SELECT id, url, `is_malicious_source` FROM `ads_yunsec_ti_url_all_df` where curdate() < date_add(modify_time, interval expire_time day) ";
+ }
+
+ @Override
+ public Map<String, Object> getRow(String ip) {
+ Integer value = intValueKV.get(ip);
+ if (value == null) {
+ return null;
+ }
+ Map<String, Object> row = new HashMap<>();
+
+ row.put("is_malicious_source", getNumBitValue(value, 0));
+ return row;
+ }
+
+ @Override
+ public String getKeyName() {
+ return this.keyName;
+ }
+
+ @Override
+ public String getTableName() {
+ return "ads_yunsec_ti_url_all_df";
+ }
+
+ @Override
+ protected void doProccRows(IntValueKV intValueKV, List<Map<String, Object>> rows, int index) {
+ for (Map<String, Object> row : rows) {
+ String ip = (String)row.get(keyName);
+ if (ip == null) {
+ LOG.warn("load Intelligence exception ,the ip is null");
+ continue;
+ }
+ List<String> values = new ArrayList<>();
+ values.add((String)row.get("is_malicious_source"));
+ int value = createInt(values);
+ synchronized (this) {
+ intValueKV.put(ip, value);
+ }
+
+ }
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/AbstractDim.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/AbstractDim.java
new file mode 100644
index 0000000..c51b0e6
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/AbstractDim.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.model;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.dim.index.IndexExecutor;
+import org.apache.rocketmq.streams.dim.index.DimIndex;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.operator.Rule;
+import org.apache.rocketmq.streams.filter.operator.expression.Expression;
+import org.apache.rocketmq.streams.filter.operator.expression.RelationExpression;
+import org.apache.rocketmq.streams.common.cache.softreference.ICache;
+import org.apache.rocketmq.streams.common.cache.softreference.impl.SoftReferenceCache;
+import org.apache.rocketmq.streams.common.configurable.BasedConfigurable;
+import org.apache.rocketmq.streams.common.cache.CompressTable;
+import org.apache.rocketmq.streams.common.utils.DataTypeUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * 这个结构代表一张表 存放表的全部数据和索引
+ */
+public abstract class AbstractDim extends BasedConfigurable {
+
+ private static final Log LOG = LogFactory.getLog(AbstractDim.class);
+
+ public static final String TYPE = "nameList";
+
+ /**
+ * 同步数据的事件间隔，单位是分钟
+ */
+ protected Long pollingTimeMintue = 60L;
+
+ /**
+ * 支持多组索引，如果一个索引是组合索引，需要拼接成一个string，用;分割 建立索引后，会创建索引的数据结构，类似Map<String,List<RowId>，可以快速定位，无索引会全表扫描，不建议使用 如有两组索引：1.name 2. ip;address
+ */
+ protected List<String> indexs = new ArrayList<>();
+
+ /**
+ * 把表数据转化成二进制存储在CompressTable中
+ */
+ protected transient volatile CompressTable dataCache;
+
+ /**
+ * 建立名单的时候，可以指定多组索引，索引的值当作key，row在datacache的index当作value，可以快速匹配索引对应的row key：索引的值 value：row在dataCache的index当作value，可以快速匹配索引对应的row
+ */
+ protected transient DimIndex nameListIndex;
+
+ //是否是唯一索引，唯一索引会用IntValueKV存储，有更高的压缩率
+ protected boolean isUniqueIndex = false;
+
+ //定时加载表数据到内存
+ protected transient ScheduledExecutorService executorService;
+
+ public AbstractDim() {
+ this.setType(TYPE);
+ }
+
+ //protected String index;//只是做标记，为了是简化indexs的赋值
+
+ public String addIndex(String... fieldNames) {
+ return addIndex(this.indexs, fieldNames);
+ }
+
+ @Override
+ protected boolean initConfigurable() {
+ boolean success = super.initConfigurable();
+ loadNameList();
+ executorService = new ScheduledThreadPoolExecutor(3);
+ executorService.scheduleWithFixedDelay(new Runnable() {
+
+ @Override
+ public void run() {
+ loadNameList();
+ }
+ }, pollingTimeMintue, pollingTimeMintue, TimeUnit.MINUTES);
+ return success;
+ }
+
+ /**
+ * 加载维表数据 创建索引
+ */
+ protected void loadNameList() {
+ try {
+ LOG.info(getConfigureName() + " begin polling data");
+ //全表数据
+ CompressTable dataCacheVar = loadData();
+ this.dataCache = dataCacheVar;
+ this.nameListIndex = buildIndex(dataCacheVar);
+ } catch (Exception e) {
+ LOG.error("Load configurables error:" + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * 给维表生成索引数据结构
+ *
+ * @param dataCacheVar 维表
+ * @return
+ */
+ protected DimIndex buildIndex(CompressTable dataCacheVar) {
+ DimIndex dimIndex = new DimIndex(this.indexs);
+ dimIndex.setUnique(isUniqueIndex);
+ dimIndex.buildIndex(dataCacheVar);
+ return dimIndex;
+ }
+
+ /**
+ * 根据索引名和索引值查询匹配的行号
+ *
+ * @param indexName
+ * @param indexValue
+ * @return
+ */
+ public List<Integer> findRowIdByIndex(String indexName, String indexValue) {
+ return nameListIndex == null ? Collections.emptyList() : nameListIndex.getRowIds(indexName, indexValue);
+ }
+
+ /**
+ * 软引用缓存，最大可能保存索引执行器，避免频繁创建，带来额外开销 同时会保护内存不被写爆，当内存不足时自动回收内存
+ */
+ private static ICache<String, IndexExecutor> cache = new SoftReferenceCache<>();
+
+ /**
+ * 先找索引，如果有索引，通过索引匹配。如果没有，全表扫表.
+ *
+ * @param expressionStr 表达式
+ * @param msg 消息
+ * @return 只返回匹配的第一行
+ */
+ public Map<String, Object> matchExpression(String expressionStr, JSONObject msg) {
+ List<Map<String, Object>> rows = matchExpression(expressionStr, msg, true, null);
+ if (rows != null && rows.size() > 0) {
+ return rows.get(0);
+ }
+ return null;
+ }
+
+ /**
+ * 先找索引，如果有索引，通过索引匹配。如果没有，全表扫表
+ *
+ * @param expressionStr 表达式
+ * @param msg 消息
+ * @return 返回全部匹配的行
+ */
+ public List<Map<String, Object>> matchExpression(String expressionStr, JSONObject msg, boolean needAll, String script) {
+ IndexExecutor indexNamelistExecutor = cache.get(expressionStr);
+ if (indexNamelistExecutor == null) {
+ indexNamelistExecutor = new IndexExecutor(expressionStr, getNameSpace(), this.indexs);
+ cache.put(expressionStr, indexNamelistExecutor);
+ }
+ if (indexNamelistExecutor.isSupport()) {
+ return indexNamelistExecutor.match(msg, this, needAll, script);
+ } else {
+ return matchExpressionByLoop(expressionStr, msg, needAll);
+ }
+ }
+
+ /**
+ * 全表扫描，做表达式匹配，返回全部匹配结果
+ *
+ * @param expressionStr
+ * @param msg
+ * @param needAll
+ * @return
+ */
+ protected List<Map<String, Object>> matchExpressionByLoop(String expressionStr, JSONObject msg, boolean needAll) {
+ CompressTable dataCache = this.dataCache;
+ List<Map<String, Object>> rows = matchExpressionByLoop(dataCache.newIterator(), expressionStr, msg, needAll);
+ return rows;
+ }
+
+ /**
+ * 全表扫描，做表达式匹配，返回全部匹配结果。join中有使用
+ *
+ * @param expressionStr
+ * @param msg
+ * @param needAll
+ * @return
+ */
+ public static List<Map<String, Object>> matchExpressionByLoop(Iterator<Map<String, Object>> it, String expressionStr, JSONObject msg, boolean needAll) {
+ List<Map<String, Object>> rows = new ArrayList<>();
+ while (it.hasNext()) {
+ Map<String, Object> values = it.next();
+ Rule ruleTemplete = ExpressionBuilder.createRule("tmp", "tmpRule", expressionStr);
+ Rule rule = ruleTemplete.copy();
+ Map<String, Expression> expressionMap = new HashMap<>();
+ for (Expression expression : rule.getExpressionMap().values()) {
+ expressionMap.put(expression.getConfigureName(), expression);
+ if (RelationExpression.class.isInstance(expression)) {
+ continue;
+ }
+ Object object = expression.getValue();
+ if (object != null && DataTypeUtil.isString(object.getClass())) {
+ String fieldName = (String)object;
+ Object value = values.get(fieldName);
+ if (value != null) {
+ Expression e = expression.copy();
+ e.setValue(value.toString());
+ expressionMap.put(e.getConfigureName(), e);
+ }
+ }
+ }
+ rule.setExpressionMap(expressionMap);
+ boolean matched = rule.execute(msg);
+ if (matched) {
+ rows.add(values);
+ if (needAll == false) {
+ return rows;
+ }
+ }
+ }
+ return rows;
+ }
+
+ protected abstract CompressTable loadData();
+
+ @Override
+ public void destroy() {
+ super.destroy();
+ executorService.shutdown();
+ }
+
+ /**
+ * 设置索引
+ *
+ * @param indexs 字段名称，多个字段";"分隔
+ */
+ public void setIndex(String indexs) {
+ if (StringUtil.isEmpty(indexs)) {
+ return;
+ }
+ List<String> tmp = new ArrayList<>();
+ String[] values = indexs.split(";");
+ this.addIndex(tmp, values);
+ this.indexs = tmp;
+ }
+
+ /**
+ * 建议指定索引，会基于索引建立map，对于等值的判断，可以快速匹配
+ *
+ * @param fieldNames
+ */
+ private String addIndex(List<String> indexs, String... fieldNames) {
+ if (fieldNames == null) {
+ return null;
+ }
+ Arrays.sort(fieldNames);
+ String index = MapKeyUtil.createKey(fieldNames);
+ if (StringUtil.isNotEmpty(index)) {
+ indexs.add(index);
+ }
+ return index;
+ }
+
+ public Long getPollingTimeMintue() {
+ return pollingTimeMintue;
+ }
+
+ public void setPollingTimeMintue(Long pollingTimeMintue) {
+ this.pollingTimeMintue = pollingTimeMintue;
+ }
+
+ public List<String> getIndexs() {
+ return indexs;
+ }
+
+ public void setIndexs(List<String> indexs) {
+ this.indexs = indexs;
+ }
+
+ public CompressTable getDataCache() {
+ return dataCache;
+ }
+
+ public boolean isUniqueIndex() {
+ return isUniqueIndex;
+ }
+
+ public void setUniqueIndex(boolean uniqueIndex) {
+ isUniqueIndex = uniqueIndex;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/BooleanFieldDBDim.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/BooleanFieldDBDim.java
new file mode 100644
index 0000000..80dd2d2
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/BooleanFieldDBDim.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.model;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.metadata.MetaData;
+import org.apache.rocketmq.streams.common.metadata.MetaDataField;
+
+/**
+ * 类似情报表，除了一个核心比对字段和主键，其他都是boolean类型的/或者是只有0/1两个值的字符串和int。可以用这个存储结构
+ */
+public class BooleanFieldDBDim extends DBDim {
+
+ private static final Log LOG = LogFactory.getLog(BooleanFieldDBDim.class);
+
+ /**
+ * 类似情报表，除了一个核心比对字段和主键，其他都是boolean类型的。可以用这个存储结构
+ *
+ * @param metaData
+ * @param notBooleanFieldName
+ * @return
+ */
+ public static boolean support(MetaData metaData, String notBooleanFieldName) {
+ List<MetaDataField> metaDataFields = metaData.getMetaDataFields();
+ for (MetaDataField metaDataField : metaDataFields) {
+ if (metaDataField.getIsPrimary()) {
+ continue;
+ }
+ if (metaDataField.getFieldName().equals(notBooleanFieldName)) {
+ continue;
+ }
+ if (!metaDataField.getDataType().matchClass(Boolean.class)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/DBDim.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/DBDim.java
new file mode 100644
index 0000000..6180820
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/model/DBDim.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.model;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configurable.annotation.ENVDependence;
+import org.apache.rocketmq.streams.common.cache.CompressTable;
+import org.apache.rocketmq.streams.common.utils.IPUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+
+public class DBDim extends AbstractDim {
+
+ private static final Log LOG = LogFactory.getLog(DBDim.class);
+
+ private String jdbcdriver = "com.mysql.jdbc.Driver";
+
+ @ENVDependence
+ private String url;
+
+ @ENVDependence
+ private String userName;
+
+ @ENVDependence
+ private String password;
+
+ private String sql;//sql 会被定时执行
+
+ private static transient AtomicInteger nameCreator = new AtomicInteger(0);
+
+ /**
+ * 是否支持批量查找
+ */
+ protected transient Boolean supportBatch = false;
+
+ public DBDim() {
+ this.setConfigureName(MapKeyUtil.createKey(IPUtil.getLocalIdentification(), System.currentTimeMillis() + "",
+ nameCreator.incrementAndGet() + ""));
+ this.setType(TYPE);
+ }
+
+ @Override
+ protected CompressTable loadData() {
+ List<Map<String, Object>> rows = executeQuery();
+ CompressTable tableCompress = new CompressTable();
+ for (Map<String, Object> row : rows) {
+ tableCompress.addRow(row);
+ }
+ return tableCompress;
+ }
+
+ protected List<Map<String, Object>> executeQuery() {
+ JDBCDriver resource = createResouce();
+ try {
+ List<Map<String, Object>> result = resource.queryForList(sql);
+ ;
+ LOG.info("load configurable's count is " + result.size());
+ return result;
+ } finally {
+ if (resource != null) {
+ resource.destroy();
+ }
+ }
+
+ }
+
+ protected JDBCDriver createResouce() {
+ return DriverBuilder.createDriver(jdbcdriver, url, userName, password);
+ }
+
+ public void setJdbcdriver(String jdbcdriver) {
+ this.jdbcdriver = jdbcdriver;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public void setSql(String sql) {
+ this.sql = sql;
+ }
+
+ public String getJdbcdriver() {
+ return jdbcdriver;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getSql() {
+ return sql;
+ }
+
+ public Boolean getSupportBatch() {
+ return supportBatch;
+ }
+
+ public void setSupportBatch(Boolean supportBatch) {
+ this.supportBatch = supportBatch;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/IDimService.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/IDimService.java
new file mode 100644
index 0000000..c5c4a2e
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/IDimService.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.service;
+
+import java.util.List;
+import java.util.Map;
+
+public interface IDimService {
+
+ /**
+ * 做维表join，关系通过表达式表示，返回所有匹配的行。因为msg没有key，表达式中，以下标表示key，如0，1，2。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式（0,functionName,filedName)&（1,functionName,filedName)|（2,functionName,filedName)
+ * @param msgs 流数据
+ * @return 符合匹配条件的所有行
+ */
+ Map<String, Object> match(String dimName, String expressionStr, Object... msgs);
+
+ /**
+ * 做维表join，关系通过表达式表示，返回所有匹配的行。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式,varName是msg中的key名称（varName,functionName,filedName)&（varName,functionName,filedName)|（varName,functionName,filedName)
+ * @param msgs 流数据
+ * @return 符合匹配条件的所有行
+ */
+ List<Map<String, Object>> matchSupportMultiRow(String dimName, String expressionStr, Map<String, Object> msgs);
+
+ /**
+ * 做维表join，关系通过表达式表示，返回匹配的一行数据，如果有多行匹配，只返回第一行。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式,varName是msg中的key名称（varName,functionName,filedName)&（varName,functionName,filedName)|（varName,functionName,filedName)
+ * @param msgs 流数据
+ * @return 返回匹配的一行数据，如果有多行匹配，只返回第一行。
+ */
+ Map<String, Object> match(String dimName, String expressionStr, Map<String, Object> msgs);
+
+ /**
+ * 做维表join，关系通过表达式表示，返回匹配的全部数据。
+ *
+ * @param dimName 维表的名称
+ * @param expressionStr 表达式,varName是msg中的key名称（varName,functionName,filedName)&（varName,functionName,filedName)|（varName,functionName,filedName)
+ * @param msgs 流数据
+ * @param script 对维表字段做处理的函数,在执行表达式前需要先对维表字段做处理，如fiedlName=trim(fieldName)
+ * @return 返回匹配的一行数据，如果有多行匹配，只返回第一行。
+ */
+ List<Map<String, Object>> matchSupportMultiRow(String dimName,
+ String expressionStr, Map<String, Object> msgs, String script);
+}
diff --git a/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/impl/DimServiceImpl.java b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/impl/DimServiceImpl.java
new file mode 100644
index 0000000..20f22ae
--- /dev/null
+++ b/rocketmq-streams-dim/src/main/java/org/apache/rocketmq/streams/dim/service/impl/DimServiceImpl.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.dim.service.impl;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.configurable.ConfigurableComponent;
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.dim.service.IDimService;
+
+public class DimServiceImpl implements IDimService {
+ protected ConfigurableComponent configurableComponent;
+
+ public DimServiceImpl(ConfigurableComponent configurableComponent) {
+ this.configurableComponent = configurableComponent;
+ }
+
+ /**
+ * 传入要比对的字段，进行规则匹配。字段和名单的比对逻辑，写在规则中
+ *
+ * @param msgs 字段名默认为数组的索引，如1，2，3
+ * @return
+ */
+ @Override
+ public Map<String, Object> match(String dimName, String expressionStr, Object... msgs) {
+ if (msgs == null || msgs.length == 0) {
+ return null;
+ }
+ int i = 0;
+ JSONObject jsonObject = new JSONObject();
+ for (Object o : msgs) {
+ jsonObject.put(i + "", o);
+ i++;
+ }
+ return match(dimName, expressionStr, jsonObject);
+ }
+
+ @Override
+ public List<Map<String, Object>> matchSupportMultiRow(String dimName, String expressionStr, Map<String, Object> msgs) {
+ return matchSupportMultiRow(dimName, expressionStr, msgs, null);
+ }
+
+ @Override
+ public List<Map<String, Object>> matchSupportMultiRow(String dimName, String expressionStr, Map<String, Object> msgs, String script) {
+ JSONObject jsonObject = createJsonable(msgs);
+ AbstractDim nameList = configurableComponent.queryConfigurable(AbstractDim.TYPE, dimName);
+ if (nameList != null) {
+ return nameList.matchExpression(expressionStr, jsonObject, true, script);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Map<String, Object> match(String nameListName, String expressionStr, Map<String, Object> parameters) {
+ JSONObject jsonObject = createJsonable(parameters);
+ AbstractDim nameList = configurableComponent.queryConfigurable(AbstractDim.TYPE, nameListName);
+ if (nameList != null) {
+ return nameList.matchExpression(expressionStr, jsonObject);
+ } else {
+ return null;
+ }
+ }
+
+ private JSONObject createJsonable(Map<String, Object> parameters) {
+ JSONObject jsonObject = null;
+ if (parameters instanceof JSONObject) {
+ jsonObject = (JSONObject)parameters;
+ } else {
+ jsonObject.putAll(parameters);
+ }
+ return jsonObject;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/ConfigureLoaderTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ConfigureLoaderTest.java
new file mode 100644
index 0000000..3f43a2a
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ConfigureLoaderTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class ConfigureLoaderTest {
+
+ @Test
+ public void testLoadResource() throws IOException {
+ // BufferedReader br = new BufferedReader(new InputStreamReader(RuleEngineRunner.class.getClassLoader
+ // ().getResourceAsStream(".")));
+ // String line = br.readLine();
+ // while (line != null) {
+ // System.out.println(line);
+ // line = br.readLine();
+ // }
+ }
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/ExpressionExecutorTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ExpressionExecutorTest.java
new file mode 100644
index 0000000..380952f
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/ExpressionExecutorTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.rocketmq.streams.filter.builder.ExpressionBuilder;
+import org.apache.rocketmq.streams.filter.operator.expression.SimpleExpression;
+import org.apache.rocketmq.streams.filter.FilterComponent;
+import org.junit.Test;
+
+import java.io.File;
+
+public class ExpressionExecutorTest {
+ private static final String CREDIBLE_PROPERTIES = "credible" + File.separator + "credible.properties";
+ private FilterComponent filterComponent;
+
+ private String namespace = "yundun.credible.net.vistor";
+ private String ruleNameSpace = "credible.rule.net.vistor";
+ private String selectorName = "credible.selector.net.vistor";
+ private String selectorExpression = "(host_uuid,=,0a2153e2-e45c-403f-8d5f-d811f400c3fb)";
+ private String procWriteList = "credible.namelist.proc";
+ private String netWriteList = "credible.namelist.net.vistor";
+
+ private String ruleExpression =
+ "(proc_path,in_resouce," + namespace + "->" + procWriteList + ")&(inner_message,not_in_expression_resouce,'"
+ + namespace + "->" + netWriteList
+ + "->(visitor_ip,=,dest_ip)&(visitor_port,=,dest_port)&(proc_path,=,program_path)')";
+
+ // @Test
+ // public void parseExpression() {
+ // List<Expression> expressions = new ArrayList<>();
+ // List<RelationExpression> relationExpressions = new ArrayList<>();
+ // Expression expression = ExpressionBuilder.createExpression("namespace", ruleExpression,
+ // expressions,
+ // relationExpressions);
+ // }
+
+ public ExpressionExecutorTest() {
+ // FilterComponent filterComponent= new FilterComponent();
+ // filterComponent.init(CREDIBLE_PROPERTIES);
+ // filterComponent.start(null);
+ // this.filterComponent=filterComponent;
+ }
+
+ @Test
+ public void testExecutor() {
+ System.out.println("hello wolrd");
+ JSONObject msg = new JSONObject();
+ msg.put("ip", "1.1.1.1");
+ boolean match = ExpressionBuilder.executeExecute(new SimpleExpression("ip", "=", "1.1.1.1"), msg);
+ System.out.println(match);
+ }
+
+ @Test
+ public void testRelationExecutor() {
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("ip", "1.2.2.3");
+ jsonObject.put("uid", "1224");
+ jsonObject.put("vmip", "1.1.1.1");
+
+ boolean value =
+ ExpressionBuilder.executeExecute("namespace", "(ip,=,1.2.2.3)&((uid,=,12214)|(vmip,=,1.1.11.1))",
+ jsonObject);
+ System.out.println(value);
+ }
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/JsonParserTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/JsonParserTest.java
new file mode 100644
index 0000000..8f58a04
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/JsonParserTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import org.junit.Test;
+
+public class JsonParserTest {
+
+ @Test
+ public void testJson() {
+ String array =
+ "[{\"value\":\"groupname\",\"key\":\"group_name\"},{\"value\":\"username\",\"key\":\"user_name\"},"
+ + "{\"value\":\"seq\",\"key\":\"index\"},{\"value\":\"egroupid\",\"key\":\"egroup_id\"},"
+ + "{\"value\":\"filepath\",\"key\":\"file_path\"},{\"value\":\"groupid\",\"key\":\"group_id\"},"
+ + "{\"value\":\"pfilename\",\"key\":\"pfile_path\"},{\"value\":\"safe_mode\",\"key\":\"perm\"},"
+ + "{\"value\":\"cmdline\",\"key\":\"cmd_line\"}]";
+ String jsonStr =
+ "{\"className\":\"com.aliyun.filter.result.FieldReNameScript\",\"oldField2NewFiled\":" + array + "}";
+ JSONArray jsonObject = JSON.parseArray(array);
+ JSONObject js = JSON.parseObject(jsonStr);
+ System.out.println(js.toJSONString());
+ }
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/NameListFunctionTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/NameListFunctionTest.java
new file mode 100644
index 0000000..80d044c
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/NameListFunctionTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.fastjson.JSONObject;
+
+import org.apache.rocketmq.streams.dim.model.AbstractDim;
+import org.apache.rocketmq.streams.dim.model.DBDim;
+import org.junit.Test;
+
+public class NameListFunctionTest {
+
+ @Test
+ public void testNameList() {
+ AbstractDim nameList = create();
+ JSONObject msg = new JSONObject();
+ msg.put("ip", "47.105.77.144");
+ msg.put("vpcId", "1");
+ msg.put("now", "2019-07-18 17:33:29");
+ long start = System.currentTimeMillis();
+ }
+
+ @Test
+ public void testNameList2() {
+ AbstractDim nameList = createMapping();
+ JSONObject msg = new JSONObject();
+ msg.put("levelFile", "aegis-vul_record:level");
+ msg.put("levelValue", "high");
+ msg.put("now", "2019-07-18 17:33:29");
+ long start = System.currentTimeMillis();
+ }
+
+ private AbstractDim create() {
+ DBDim dbNameList = new DBDim();
+ dbNameList.setNameSpace("soc");
+ dbNameList.setConfigureName("isoc_field_mappings");
+ dbNameList.setUrl("");
+ dbNameList.setUserName("");
+ dbNameList.setPassword("");
+ dbNameList.setSql("SELECT * FROM `ecs_info` WHERE STATUS=1 LIMIT 1");
+ List<String> ipFieldNames = new ArrayList<>();
+ ipFieldNames.add("public_ips");
+ ipFieldNames.add("inner_ips");
+ ipFieldNames.add("eip");
+ ipFieldNames.add("private_ips");
+ dbNameList.init();
+ return dbNameList;
+ }
+
+ @Test
+ public void testNameListAllRow() {
+ AbstractDim nameList = createMapping();
+ JSONObject msg = new JSONObject();
+ msg.put("levelFile", "aegis-vul_record:level");
+ msg.put("levelValue", "high");
+ msg.put("now", "2019-07-18 17:33:29");
+ long start = System.currentTimeMillis();
+
+ }
+
+ private AbstractDim createMapping() {
+ DBDim dbNameList = new DBDim();
+ dbNameList.setNameSpace("soc");
+ dbNameList.setConfigureName("isoc_field_mappings");
+ dbNameList.setUrl("");
+ dbNameList.setUserName("");
+ dbNameList.setPassword("");
+ dbNameList.setSql("select * from ads_yunsec_ti_url_all_df limit 100000");
+ dbNameList.init();
+ return dbNameList;
+ }
+
+}
diff --git a/rocketmq-streams-dim/src/test/java/com/aliyun/service/TableCompressTest.java b/rocketmq-streams-dim/src/test/java/com/aliyun/service/TableCompressTest.java
new file mode 100644
index 0000000..3c89fd6
--- /dev/null
+++ b/rocketmq-streams-dim/src/test/java/com/aliyun/service/TableCompressTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.aliyun.service;
+
+import org.junit.Test;
+
+public class TableCompressTest {
+
+ @Test
+ public void testNameList() throws InterruptedException {
+ }
+}
diff --git a/rocketmq-streams-lease/pom.xml b/rocketmq-streams-lease/pom.xml
new file mode 100755
index 0000000..a0ad067
--- /dev/null
+++ b/rocketmq-streams-lease/pom.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>rocketmq-streams-lease</artifactId>
+ <name>ROCKETMQ STREAMS :: lease</name>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-channel-db</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-streams-db-operator</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/LeaseComponent.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/LeaseComponent.java
new file mode 100644
index 0000000..3a527a3
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/LeaseComponent.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */package org.apache.rocketmq.streams.lease;
+
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.common.component.AbstractComponent;
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.component.ConfigureDescriptor;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.configurable.service.ConfigurableServcieType;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+import org.apache.rocketmq.streams.lease.service.ILeaseStorage;
+import org.apache.rocketmq.streams.lease.service.impl.LeaseServiceImpl;
+import org.apache.rocketmq.streams.lease.service.impl.MockLeaseImpl;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.lease.service.storages.DBLeaseStorage;
+import org.apache.rocketmq.streams.serviceloader.ServiceLoaderComponent;
+
+/**
+ * 通过db实现租约和锁，可以更轻量级，减少其他中间件的依赖 使用主备场景，只有一个实例运行，当当前实例挂掉，在一定时间内，会被其他实例接手 也可以用于全局锁
+ *
+ * @date 1/9/19
+ */
+public class LeaseComponent extends AbstractComponent<ILeaseService> {
+
+ private static LeaseComponent leaseComponent = null;
+ private static final Log LOG = LogFactory.getLog(LeaseComponent.class);
+ private ILeaseService leaseService;
+
+ public LeaseComponent() {
+ initConfigurableServiceDescriptor();
+ addConfigureDescriptor(
+ new ConfigureDescriptor(CONNECT_TYPE, false, ConfigurableServcieType.DEFAULT_SERVICE_NAME));
+ }
+
+ public static LeaseComponent getInstance() {
+ if(leaseComponent==null){
+ synchronized (LeaseComponent.class){
+ if(leaseComponent==null){
+ leaseComponent =ComponentCreator.getComponent(null,LeaseComponent.class);
+ }
+ }
+ }
+ return leaseComponent;
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ @Override
+ public ILeaseService getService() {
+ return leaseService;
+ }
+
+ @Override
+ protected boolean startComponent(String namespace) {
+ return true;
+ }
+
+ @Override
+ protected boolean initProperties(Properties properties) {
+ String connectType = properties.getProperty(JDBC_URL);
+ if (StringUtil.isEmpty(connectType)) {
+ MockLeaseImpl mockLease = new MockLeaseImpl();
+ this.leaseService=mockLease;
+ return true;
+ }
+
+ LeaseServiceImpl leaseService= new LeaseServiceImpl();
+ String storageName=ComponentCreator.getProperties().getProperty(ConfigureFileKey.LEASE_STORAGE_NAME);
+ ILeaseStorage storasge=null;
+ if(StringUtil.isEmpty(storageName)){
+ String jdbc = properties.getProperty(AbstractComponent.JDBC_DRIVER);
+ String url = properties.getProperty(AbstractComponent.JDBC_URL);
+ String userName = properties.getProperty(AbstractComponent.JDBC_USERNAME);
+ String password = properties.getProperty(AbstractComponent.JDBC_PASSWORD);
+ storasge=new DBLeaseStorage(jdbc,url,userName,password);
+ }else {
+ storasge= (ILeaseStorage)ServiceLoaderComponent.getInstance(ILeaseStorage.class).loadService(storageName);
+ }
+ leaseService.setLeaseStorage(storasge);
+ this.leaseService=leaseService;
+ return true;
+ }
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/model/LeaseInfo.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/model/LeaseInfo.java
new file mode 100644
index 0000000..469a711
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/model/LeaseInfo.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.model;
+
+/**
+ * 租约对象，需要创建租约表。
+ */
+
+import java.io.Serializable;
+import java.util.Date;
+
+public class LeaseInfo implements Serializable {
+ private static final long serialVersionUID = 665608838255753618L;
+ private Long id;
+ private Date createTime;
+ private Date updateTime;
+ private String leaseName;//租约名称，多个进程共享一个租约，只要名称相同即可
+ private String leaseUserIp;//区分不同的租约实体，以前默认用ip，但一个机器多个进程的情况下，用ip会区分不开，后续会加上进程号
+ private Date leaseEndDate;//租约到期时间
+ private int status;//租约的有效状态
+ private long version;//版本，通过版本保证更新原子性
+
+ public LeaseInfo() {
+ }
+
+ /**
+ * 建表语句
+ *
+ * @return
+ */
+ public static String createTableSQL() {
+ return "CREATE TABLE IF NOT EXISTS `lease_info` (\n"
+ + " `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',\n"
+ + " `gmt_create` datetime NOT NULL COMMENT '创建时间',\n"
+ + " `gmt_modified` datetime NOT NULL COMMENT '修改时间',\n"
+ + " `lease_name` varchar(255) NOT NULL COMMENT '租约名称',\n"
+ + " `lease_user_ip` varchar(255) NOT NULL COMMENT '租者IP',\n"
+ + " `lease_end_time` varchar(255) NOT NULL COMMENT '租约到期时间',\n"
+ + " `status` int(11) NOT NULL DEFAULT '1' COMMENT '状态',\n"
+ + " `version` bigint(20) NOT NULL COMMENT '版本',\n"
+ + " `candidate_lease_ip` varchar(255) DEFAULT NULL COMMENT '候选租约ip',\n"
+ + " PRIMARY KEY (`id`),\n"
+ + " UNIQUE KEY `uk_name` (`lease_name`)\n"
+ + ") ENGINE=InnoDB AUTO_INCREMENT=8150 DEFAULT CHARSET=utf8 COMMENT='租约信息'\n"
+ + ";";
+ }
+
+ public Long getId() {
+ return this.id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public Date getCreateTime() {
+ return this.createTime;
+ }
+
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ public Date getUpdateTime() {
+ return this.updateTime;
+ }
+
+ public void setUpdateTime(Date updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ public String getLeaseUserIp() {
+ return this.leaseUserIp;
+ }
+
+ public void setLeaseUserIp(String leaseUserIp) {
+ this.leaseUserIp = leaseUserIp;
+ }
+
+ public Date getLeaseEndDate() {
+ return this.leaseEndDate;
+ }
+
+ public void setLeaseEndDate(Date leaseEndDate) {
+ this.leaseEndDate = leaseEndDate;
+ }
+
+ public int getStatus() {
+ return this.status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public String getLeaseName() {
+ return this.leaseName;
+ }
+
+ public void setLeaseName(String leaseName) {
+ this.leaseName = leaseName;
+ }
+
+ public long getVersion() {
+ return this.version;
+ }
+
+ public void setVersion(long version) {
+ this.version = version;
+ }
+
+}
+
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseGetCallback.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseGetCallback.java
new file mode 100644
index 0000000..ad63dde
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseGetCallback.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service;
+
+import java.util.Date;
+
+public interface ILeaseGetCallback {
+
+ /**
+ * 当成功获取租约时，回调接口
+ *
+ * @param nextLeaseDate 租约到期时间
+ */
+ void callback(Date nextLeaseDate);
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseService.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseService.java
new file mode 100644
index 0000000..7782c52
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseService.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service;
+
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+
+/**
+ * 通过db实现租约和锁，可以更轻量级，减少其他中间件的依赖 使用主备场景，只有一个实例运行，当当前实例挂掉，在一定时间内，会被其他实例接手 也可以用于全局锁
+ */
+public interface ILeaseService {
+
+ /**
+ * 默认锁定时间
+ */
+ static final int DEFALUT_LOCK_TIME = 60 * 5;
+
+ /**
+ * 检查某用户当前时间是否具有租约。这个方法是纯内存操作，无性能开销
+ *
+ * @return true，租约有效；false，租约无效
+ */
+ boolean hasLease(String name);
+
+ /**
+ * 申请租约，会启动一个线程，不停申请租约，直到申请成功。 申请成功后，每 租期/2 续约。 如果目前被其他租户获取租约，只有在对方租约失效，后才允许新的租户获取租约
+ *
+ * @param name 租约名称，无特殊要求，相同名称会竞争租约
+ */
+ void startLeaseTask(String name);
+
+ /**
+ * 申请租约，会启动一个线程，不停申请租约，直到申请成功。 申请成功后，每 租期/2 续约。 如果目前被其他租户获取租约，只有在对方租约失效，后才允许新的租户获取租约
+ *
+ * @param name 租约名称，无特殊要求，相同名称会竞争租约
+ * @param callback 当第一获取租约时，回调此函数
+ */
+ void startLeaseTask(final String name, ILeaseGetCallback callback);
+
+ /**
+ * 申请租约，会启动一个线程，不停申请租约，直到申请成功。 申请成功后，每 租期/2 续约。 如果目前被其他租户获取租约，只有在对方租约失效，后才允许新的租户获取租约
+ *
+ * @param name 租约名称，无特殊要求，相同名称会竞争租约
+ * @param leaseTermSecond 租期，在租期内可以做业务处理，单位是秒
+ * @param callback 当第一获取租约时，回调此函数
+ */
+ void startLeaseTask(final String name, int leaseTermSecond, ILeaseGetCallback callback);
+
+ /**
+ * 申请锁,无论成功与否，立刻返回。如果不释放，最大锁定时间是5分钟
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @return 是否枷锁成功
+ */
+ boolean lock(String name, String lockerName);
+
+ /**
+ * 申请锁,无论成功与否，立刻返回。默认锁定时间是5分钟
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param lockTimeSecond 如果不释放，锁定的最大时间，单位是秒
+ * @return 是否枷锁成功
+ * @return
+ */
+ boolean lock(String name, String lockerName, int lockTimeSecond);
+
+ /**
+ * 申请锁，如果没有则等待，等待时间可以指定，如果是－1 则无限等待。如果不释放，最大锁定时间是5分钟
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param waitTime 没获取锁时，最大等待多长时间，如果是－1 则无限等待
+ * @return 是否枷锁成功
+ */
+ boolean tryLocker(String name, String lockerName, long waitTime);
+
+ /**
+ * 申请锁，如果没有则等待，等待时间可以指定，如果是－1 则无限等待。如果不释放，最大锁定时间是lockTimeSecond
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param waitTime 没获取锁时，最大等待多长时间，如果是－1 则无限等待
+ * @param lockTimeSecond 如果不释放，锁定的最大时间，单位是秒
+ * @return 是否枷锁成功
+ */
+ boolean tryLocker(String name, String lockerName, long waitTime, int lockTimeSecond);
+
+ /**
+ * 释放锁
+ *
+ * @param name
+ * @param lockerName
+ * @return
+ */
+ boolean unlock(String name, String lockerName);
+
+ /**
+ * 对于已经获取锁的，可以通过这个方法，一直持有锁。 和租约的区别是，当释放锁后，无其他实例抢占。无法实现主备模式
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param lockTimeSecond 租期，这个方法会自动续约，如果不主动释放，会一直持有锁
+ * @return 是否成功获取锁
+ */
+ boolean holdLock(String name, String lockerName, int lockTimeSecond);
+
+ /**
+ * 是否持有锁，不会申请锁。如果以前申请过，且未过期，返回true，否则返回false
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @return
+ */
+ boolean hasHoldLock(String name, String lockerName);
+
+ List<LeaseInfo> queryLockedInstanceByNamePrefix(String name, String lockerNamePrefix);
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorage.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorage.java
new file mode 100644
index 0000000..9278bc0
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorage.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+
+public interface ILeaseStorage {
+
+ /**
+ * 更新lease info，需要是原子操作，存储保障多线程操作的原子性
+ *
+ * @param leaseInfo 租约表数据
+ * @return
+ */
+ boolean updateLeaseInfo(LeaseInfo leaseInfo);
+
+ /**
+ * 统计这个租约名称下，LeaseInfo对象个数
+ *
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ Integer countLeaseInfo(String leaseName);
+
+ /**
+ * 查询无效的的租约
+ *
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryInValidateLease(String leaseName);
+
+ /**
+ * 查询有效的的租约
+ *
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryValidateLease(String leaseName);
+
+ /**
+ * 按前缀查询有效的租约信息
+ *
+ * @param namePrefix
+ * @return
+ */
+ List<LeaseInfo> queryValidateLeaseByNamePrefix(String namePrefix);
+
+ /**
+ * 增加租约
+ *
+ * @param leaseInfo 租约名称，无特殊要求，相同名称会竞争租约
+ */
+ void addLeaseInfo(LeaseInfo leaseInfo);
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorasge.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorasge.java
new file mode 100644
index 0000000..cbfe26e
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/ILeaseStorasge.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.streams.lease.service;
+
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+
+public interface ILeaseStorasge {
+
+
+
+ /**
+ * 更新lease info，需要是原子操作，存储保障多线程操作的原子性
+ * @param leaseInfo 租约表数据
+ * @return
+ */
+ boolean updateLeaseInfo(LeaseInfo leaseInfo);
+
+ /**
+ * 统计这个租约名称下，LeaseInfo对象个数
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ Integer countLeaseInfo(String leaseName);
+
+ /**
+ * 查询无效的的租约
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryInValidateLease(String leaseName);
+
+
+ /**
+ * 查询无效的的租约
+ * @param leaseName 租约名称，无特殊要求，相同名称会竞争租约
+ * @return
+ */
+ LeaseInfo queryValidateLease(String leaseName);
+
+ /**
+ * 增加租约
+ * @param leaseInfo 租约名称，无特殊要求，相同名称会竞争租约
+ */
+ void addLeaseInfo(LeaseInfo leaseInfo);
+
+
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/BasedLesaseImpl.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/BasedLesaseImpl.java
new file mode 100644
index 0000000..21db98d
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/BasedLesaseImpl.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.impl;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.IPUtil;
+import org.apache.rocketmq.streams.common.utils.RuntimeUtil;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+import org.apache.rocketmq.streams.lease.service.ILeaseStorage;
+
+public abstract class BasedLesaseImpl implements ILeaseService {
+ private static final Log LOG = LogFactory.getLog(BasedLesaseImpl.class);
+
+ private static final String CONSISTENT_HASH_PREFIX = "consistent_hash_";
+ private static AtomicBoolean syncStart = new AtomicBoolean(false);
+ private static final int synTime = 120; // 5分钟的一致性hash同步时间太久了，改为2分钟
+ protected ScheduledExecutorService taskExecutor = null;
+ protected int leaseTerm = 300 * 2; // 租约时间
+
+ // protected transient JDBCDriver jdbcDataSource = null;
+ protected ILeaseStorage leaseStorage;
+ protected volatile Map<String, Date> leaseName2Date = new ConcurrentHashMap<>(); // 每个lease name对应的租约到期时间
+
+ public BasedLesaseImpl() {
+
+ taskExecutor = new ScheduledThreadPoolExecutor(10);
+
+ }
+
+ /**
+ * lease_name: consistent_hash_ip, lease_user_ip: ip,定时刷新lease_info表，检查一致性hash环的节点情况
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public boolean hasLease(String name) {
+ // 内存中没有租约信息则表示 没有租约
+ Date leaseEndTime = leaseName2Date.get(name);
+ if (leaseEndTime == null) {
+ // LOG.info("内存中根据 " + name + "没有查询到租约信息，表示没有租约");
+ return false;
+ }
+ // LOG.info("查询是否有租约 name:" + name + " ,当前时间：" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())
+ // + " 租约到期时间 " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(leaseEndTime));
+ // 有租约时间，并且租约时间大于当前时间，表示有租约信息
+ if (new Date().before(leaseEndTime)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private final Map<String, AtomicBoolean> startLeaseMap = new HashMap<>();
+
+ @Override
+ public void startLeaseTask(final String name) {
+ startLeaseTask(name, this.leaseTerm, null);
+ }
+
+ @Override
+ public void startLeaseTask(final String name, ILeaseGetCallback callback) {
+ startLeaseTask(name, this.leaseTerm, callback);
+ }
+
+ @Override
+ public void startLeaseTask(final String name, int leaseTerm, ILeaseGetCallback callback) {
+ ApplyTask applyTask = new ApplyTask(leaseTerm, name, callback);
+ startLeaseTask(name, applyTask, leaseTerm / 2, true);
+ }
+
+ /**
+ * 启动定时器，定时执行任务，确保任务可重入
+ *
+ * @param name
+ * @param runnable 具体任务
+ * @param scheduleTime 调度时间
+ * @param startNow 是否立刻启动一次
+ */
+ protected void startLeaseTask(final String name, Runnable runnable, int scheduleTime, boolean startNow) {
+ AtomicBoolean isStartLease = startLeaseMap.get(name);//多次调用，只启动一次定时任务
+ if (isStartLease == null) {
+ synchronized (this) {
+ isStartLease = startLeaseMap.get(name);
+ if (isStartLease == null) {
+ isStartLease = new AtomicBoolean(false);
+ startLeaseMap.put(name, isStartLease);
+ }
+ }
+ }
+ if (isStartLease.compareAndSet(false, true)) {
+ if (startNow) {
+ runnable.run();
+ }
+ taskExecutor.scheduleWithFixedDelay(runnable, 0, scheduleTime, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * 续约任务
+ */
+ protected class ApplyTask implements Runnable {
+
+ protected String name;
+ protected int leaseTerm;
+ protected ILeaseGetCallback callback;
+
+ public ApplyTask(int leaseTerm, String name) {
+ this(leaseTerm, name, null);
+ }
+
+ public ApplyTask(int leaseTerm, String name, ILeaseGetCallback callback) {
+ this.name = name;
+ this.leaseTerm = leaseTerm;
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // LOG.info("LeaseServiceImpl name: " + name + "开始获取租约...");
+ AtomicBoolean newApplyLease = new AtomicBoolean(false);
+ Date leaseDate = applyLeaseTask(leaseTerm, name, newApplyLease);
+ if (leaseDate != null) {
+ leaseName2Date.put(name, leaseDate);
+ LOG.info("LeaseServiceImpl, name: " + name + " " + getSelfUser() + " 获取租约成功, 租约到期时间为 "
+ + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(leaseDate));
+ } else {
+ // fix.2020.08.13 这时name对应的租约可能还在有效期内,或者本机还持有租约，需要remove
+ // leaseName2Date.remove(name);
+ LOG.info("LeaseServiceImpl name: " + name + " " + getSelfUser() + " 获取租约失败 ");
+ }
+ if (newApplyLease.get() && callback != null) {
+ callback.callback(leaseDate);
+ }
+ } catch (Exception e) {
+ LOG.error(" LeaseServiceImpl name: " + name + " " + getSelfUser() + " 获取租约出现异常 ", e);
+ }
+
+ }
+ }
+
+ /**
+ * 申请租约，如果当期租约有效，直接更新一个租约周期，如果当前租约无效，先查询是否有有效的租约，如果有申请失败，否则直接申请租约
+ */
+ protected Date applyLeaseTask(int leaseTerm, String name, AtomicBoolean newApplyLease) {
+
+ // 计算下一次租约时间 = 当前时间 + 租约时长
+ Date nextLeaseDate = DateUtil.addSecond(new Date(), leaseTerm);
+
+ // 1 如果已经有租约，则更新租约时间(内存和数据库)即可
+ if (hasLease(name)) {
+ // LOG.info("用户已有租约，更新数据库和内存中的租约信息");
+ // 更新数据库
+ LeaseInfo leaseInfo = queryValidateLease(name);
+ if (leaseInfo == null) {
+ LOG.error("LeaseServiceImpl applyLeaseTask leaseInfo is null");
+ return null;
+ }
+ // fix.2020.08.13,与本机ip相等且满足一致性hash分配策略，才续约，其他情况为null
+ String leaseUserIp = leaseInfo.getLeaseUserIp();
+ if (!leaseUserIp.equals(getSelfUser())) {
+ return null;
+ }
+ leaseInfo.setLeaseEndDate(nextLeaseDate);
+ updateLeaseInfo(leaseInfo);
+ return nextLeaseDate;
+ }
+
+ // 2 没有租约情况 判断是否可以获取租约，只要租约没有被其他人获取，则说明有有效租约
+ boolean success = canGetLease(name);
+ if (!success) { // 表示被其他机器获取到了有效的租约
+ // LOG.info("其他机器获取到了有效的租约");
+ return null;
+ }
+
+ // 3 没有租约而且可以获取租约的情况，则尝试使用数据库原子更新的方式获取租约，保证只有一台机器成功获取租约，而且可以运行
+ boolean flag = tryGetLease(name, nextLeaseDate);
+ if (flag) { // 获取租约成功
+ newApplyLease.set(true);
+ return nextLeaseDate;
+ }
+ return null;
+
+ }
+
+ /**
+ * 查询数据库，自己是否在租期内或没有被其他人租用
+ *
+ * @return
+ */
+ protected boolean canGetLease(String name) {
+ LeaseInfo leaseInfo = queryValidateLease(name);
+ if (leaseInfo == null) {
+ return true;
+ }
+ // fix.2020.08.13,租约ip为本机ip，且与一致性hash分配ip一致，才是有效租约
+ String leaseUserIp = leaseInfo.getLeaseUserIp();
+ if (leaseUserIp.equals(getSelfUser())) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * 更新数据库，占用租期并更新租期时间
+ *
+ * @param time
+ */
+ protected boolean tryGetLease(String name, Date time) {
+ // LOG.info("尝试获取租约 lease name is : " + name + " 下次到期时间: "
+ // + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(time));
+ LeaseInfo validateLeaseInfo = queryValidateLease(name);
+
+ if (validateLeaseInfo == null) {// 这里有两种情况 1 数据库里面没有租约信息 2 数据库里面有租约信息但是已经过期
+ Integer count = countLeaseInfo(name);
+ if (count == null || count == 0) {// 表示现在数据库里面没有任何租约信息，插入租约成功则表示获取成功，失败表示在这一时刻其他机器获取了租约
+ // LOG.info("数据库中暂时没有租约信息，尝试原子插入租约:" + name);
+ // fix.2020.08.13,经过一致性hash计算，该名字的任务不应该在本机执行，直接返回，无需插入。只有分配到hash执行权限的机器才可以插入并获取租约
+ if (!getSelfUser().equals(getConsistentHashHost(name))) {
+ return false;
+ }
+ validateLeaseInfo = new LeaseInfo();
+ validateLeaseInfo.setLeaseName(name);
+ validateLeaseInfo.setLeaseUserIp(getSelfUser());
+ validateLeaseInfo.setLeaseEndDate(time);
+ validateLeaseInfo.setStatus(1);
+ validateLeaseInfo.setVersion(1);
+ if (insert(validateLeaseInfo)) {
+ LOG.info("数据库中暂时没有租约信息，原子插入成功，获取租约成功:" + name);
+ return true;
+ } else {
+ LOG.info("数据库中暂时没有租约信息，原子插入失败，已经被其他机器获取租约:" + name);
+ return false;
+ }
+ } else { // 表示数据库里面有一条但是无效，这里需要两台机器按照version进行原子更新，更新成功的获取租约
+ // LOG.info("数据库中有一条无效的租约信息，尝试根据版本号去原子更新租约信息:" + name);
+ LeaseInfo inValidateLeaseInfo = queryInValidateLease(name);
+ if (inValidateLeaseInfo == null) {// 说明这个时候另外一台机器获取成功了
+ LOG.info("另外一台机器获取成功了租约:" + name);
+ return false;
+ }
+ // fix.2020.08.13,机器重启之后，该名字的任务已经不分配在此机器上执行，直接返回，无需更新数据库
+ if (!getSelfUser().equals(getConsistentHashHost(name))) {
+ return false;
+ }
+ inValidateLeaseInfo.setLeaseName(name);
+ inValidateLeaseInfo.setLeaseUserIp(getSelfUser());
+ inValidateLeaseInfo.setLeaseEndDate(time);
+ inValidateLeaseInfo.setStatus(1);
+ boolean success = updateDBLeaseInfo(inValidateLeaseInfo);
+ if (success) {
+ LOG.info("LeaseServiceImpl 原子更新租约成功，当前机器获取到了租约信息:" + name);
+ } else {
+ LOG.info("LeaseServiceImpl 原子更新租约失败，租约被其他机器获取:" + name);
+ }
+ return success;
+ }
+
+ } else { // 判断是否是自己获取了租约，如果是自己获取了租约则更新时间（内存和数据库），
+ // 这里是为了解决机器重启的情况，机器重启，内存中没有租约信息，但是实际上该用户是有租约权限的
+ // fix.2020.08.13,租约的ip与本机ip相等，且满足一致性hash策略，才会被本机执行
+ String leaseUserIp = validateLeaseInfo.getLeaseUserIp();
+ if (leaseUserIp.equals(getSelfUser())) {
+ // 如果当期用户有租约信息，则更新数据库
+ validateLeaseInfo.setLeaseEndDate(time);
+ boolean hasUpdate = updateLeaseInfo(validateLeaseInfo);
+ if (hasUpdate) {
+ LOG.info(
+ "LeaseServiceImpl机器重启情况，当前用户有租约信息，并且更新数据库成功，租约信息为 name :" + validateLeaseInfo.getLeaseName()
+ + " ip : " + validateLeaseInfo.getLeaseUserIp() + " 到期时间 : " + new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss").format(validateLeaseInfo.getLeaseEndDate()));
+ return true;
+ } else {
+ LOG.info("LeaseServiceImpl 机器重启情况，当前用户有租约信息，并且更新数据库失败，表示失去租约:" + name);
+ return false;
+ }
+ }
+ // LOG.info("LeaseServiceImpl 租约被其他机器获取，租约信息为 name :" + validateLeaseInfo.getLeaseName() + " ip : "
+ // + validateLeaseInfo.getLeaseUserIp() + " 到期时间 : "
+ // + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(validateLeaseInfo.getLeaseEndDate()));
+ return false;
+ }
+
+ }
+
+ protected LeaseInfo queryValidateLease(String name) {
+ //String sql = "SELECT * FROM lease_info WHERE lease_name ='" + name + "' and status=1 and lease_end_time>now()";
+ //// LOG.info("LeaseServiceImpl query validate lease sql:" + sql);
+ //return queryLease(name, sql);
+ return leaseStorage.queryValidateLease(name);
+ }
+
+ protected List<LeaseInfo> queryValidateLeaseByNamePrefix(String namePrefix) {
+ return leaseStorage.queryValidateLeaseByNamePrefix(namePrefix);
+ }
+
+ /**
+ * 如果发生唯一索引冲突返回失败
+ *
+ * @param leaseInfo
+ * @return
+ */
+ private boolean insert(LeaseInfo leaseInfo) {
+ try {
+ addLeaseInfo(leaseInfo);
+ return true;
+ } catch (Exception e) {
+ LOG.error("LeaseServiceImpl insert error", e);
+ return false;
+ }
+ }
+
+ /**
+ * 更新时需要加version＝当前version，如果更新数据条数为0，返回false
+ *
+ * @param leaseInfo
+ * @return
+ */
+ protected boolean updateDBLeaseInfo(LeaseInfo leaseInfo) {
+ return updateLeaseInfo(leaseInfo);
+ }
+
+ protected boolean updateLeaseInfo(LeaseInfo leaseInfo) {
+
+ return leaseStorage.updateLeaseInfo(leaseInfo);
+ }
+
+ protected Integer countLeaseInfo(String name) {
+
+ return leaseStorage.countLeaseInfo(name);
+ }
+
+ protected LeaseInfo queryInValidateLease(String name) {
+
+ return leaseStorage.queryInValidateLease(name);
+ }
+
+ protected void addLeaseInfo(LeaseInfo leaseInfo) {
+
+ leaseStorage.addLeaseInfo(leaseInfo);
+
+ }
+
+ /**
+ * 本地ip地址作为自己的唯一标识
+ *
+ * @return
+ */
+ public static String getLocalName() {
+ return IPUtil.getLocalIdentification() + ":" + Optional.ofNullable(RuntimeUtil.getPid()).orElse("UNKNOWN");
+ }
+
+ /**
+ * 本地ip地址作为自己的唯一标识
+ *
+ * @return
+ */
+ public String getSelfUser() {
+ return getLocalName();
+ }
+
+ private String getConsistentHashHost(String name) {
+ //if (StringUtil.isEmpty(leaseConsistentHashSuffix)) {
+ // return getSelfUser();
+ //}
+ //return consistentHashInstance.getCandidateNode(name);
+ return getSelfUser();
+ }
+
+ public void setLeaseStorage(ILeaseStorage leaseStorage) {
+ this.leaseStorage = leaseStorage;
+ }
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/LeaseServiceImpl.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/LeaseServiceImpl.java
new file mode 100644
index 0000000..860710a
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/LeaseServiceImpl.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.impl;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.MapKeyUtil;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class LeaseServiceImpl extends BasedLesaseImpl {
+
+ private static final Log LOG = LogFactory.getLog(LeaseServiceImpl.class);
+
+ private transient ConcurrentHashMap<String, HoldLockTask> holdLockTasks = new ConcurrentHashMap();
+
+ protected ConcurrentHashMap<String, HoldLockFunture> seizeLockingFuntures = new ConcurrentHashMap<>();
+ //如果是抢占锁状态中，则不允许申请锁
+
+ public LeaseServiceImpl() {
+ super();
+ }
+
+ /**
+ * 尝试获取锁，可以等待waitTime，如果到点未返回，则直接返回。如果是-1，则一直等待
+ *
+ * @param name 业务名称
+ * @param lockerName 锁名称
+ * @param waitTime 等待时间，是微秒单位
+ * @return
+ */
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime) {
+ return tryLocker(name, lockerName, waitTime, ILeaseService.DEFALUT_LOCK_TIME);
+ }
+
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime, int lockTimeSecond) {
+ long now = System.currentTimeMillis();
+ boolean success = lock(name, lockerName, lockTimeSecond);
+ while (!success) {
+ if (waitTime > -1 && (System.currentTimeMillis() - now > waitTime)) {
+ break;
+ }
+ success = lock(name, lockerName, lockTimeSecond);
+ if (success) {
+ return success;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ LOG.error("LeaseServiceImpl try locker error", e);
+ }
+ }
+ return success;
+
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName) {
+ return lock(name, lockerName, ILeaseService.DEFALUT_LOCK_TIME);
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName, int leaseSecond) {
+ lockerName = createLockName(name, lockerName);
+ Future future = seizeLockingFuntures.get(lockerName);
+ if (future != null && ((HoldLockFunture)future).isDone == false) {
+ return false;
+ }
+ Date nextLeaseDate =
+ DateUtil.addSecond(new Date(), leaseSecond);// 默认锁定5分钟，用完需要立刻释放.如果时间不同步，可能导致锁失败
+ boolean success = tryGetLease(lockerName, nextLeaseDate);// 申请锁，锁的时间是leaseTerm
+ return success;
+ }
+
+ @Override
+ public boolean unlock(String name, String lockerName) {
+ // LOG.info("LeaseServiceImpl unlock,name:" + name);
+ lockerName = createLockName(name, lockerName);
+ LeaseInfo validateLeaseInfo = queryValidateLease(lockerName);
+ if (validateLeaseInfo == null) {
+ LOG.warn("LeaseServiceImpl unlock,validateLeaseInfo is null,lockerName:" + lockerName);
+ }
+ if (validateLeaseInfo != null && validateLeaseInfo.getLeaseUserIp().equals(getSelfUser())) {
+ validateLeaseInfo.setStatus(0);
+ updateDBLeaseInfo(validateLeaseInfo);
+ }
+ HoldLockTask holdLockTask = holdLockTasks.remove(lockerName);
+ if (holdLockTask != null) {
+ holdLockTask.close();
+ }
+ leaseName2Date.remove(lockerName);
+ return false;
+ }
+
+ /**
+ * 如果有锁，则一直持有，如果不能获取，则结束。和租约不同，租约是没有也会尝试重试，一备对方挂机，自己可以接手工作
+ *
+ * @param name
+ * @param secondeName
+ * @param lockTimeSecond 获取锁的时间
+ * @return
+ */
+ @Override
+ public boolean holdLock(String name, String secondeName, int lockTimeSecond) {
+ if (hasHoldLock(name, secondeName)) {
+ return true;
+ }
+ synchronized (this) {
+ if (hasHoldLock(name, secondeName)) {
+ return true;
+ }
+ String lockerName = createLockName(name, secondeName);
+ Date nextLeaseDate =
+ DateUtil.addSecond(new Date(), lockTimeSecond);
+ boolean success = tryGetLease(lockerName, nextLeaseDate);// 申请锁，锁的时间是leaseTerm
+ if (success == false) {
+ return false;
+ }
+ leaseName2Date.put(lockerName, nextLeaseDate);
+
+ if (!holdLockTasks.containsKey(lockerName)) {
+ HoldLockTask holdLockTask = new HoldLockTask(lockTimeSecond, lockerName, this);
+ holdLockTask.start();
+ holdLockTasks.putIfAbsent(lockerName, holdLockTask);
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * 是否持有锁，不访问数据库，直接看本地
+ *
+ * @param name
+ * @param secondeName
+ * @return
+ */
+ @Override
+ public boolean hasHoldLock(String name, String secondeName) {
+ String lockerName = createLockName(name, secondeName);
+ return hasLease(lockerName);
+ }
+
+ @Override
+ public List<LeaseInfo> queryLockedInstanceByNamePrefix(String name, String lockerNamePrefix) {
+ String leaseNamePrefix = MapKeyUtil.createKey(name, lockerNamePrefix);
+ return queryValidateLeaseByNamePrefix(leaseNamePrefix);
+ }
+
+ private String createLockName(String name, String lockerName) {
+ return MapKeyUtil.createKey(name, lockerName);
+ }
+
+ private class HoldLockTask extends ApplyTask {
+ protected volatile boolean iscontinue = true;
+ protected LeaseServiceImpl leaseService;
+ protected ScheduledExecutorService scheduledExecutor;
+
+ public HoldLockTask(int leaseTerm, String name, LeaseServiceImpl leaseService) {
+ super(leaseTerm, name);
+ this.leaseService = leaseService;
+ scheduledExecutor = new ScheduledThreadPoolExecutor(1);
+
+ }
+
+ public void start() {
+ scheduledExecutor.scheduleWithFixedDelay(this, leaseTerm / 2, leaseTerm / 2, TimeUnit.SECONDS);
+ }
+
+ public void close() {
+ iscontinue = false;
+ if (scheduledExecutor != null) {
+ scheduledExecutor.shutdown();
+ }
+ }
+
+ public boolean isIscontinue() {
+ return iscontinue;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (!iscontinue) {
+ return;
+ }
+ Date leaseDate = applyLeaseTask(leaseTerm, name, new AtomicBoolean(false));
+ if (leaseDate != null) {
+ leaseName2Date.put(name, leaseDate);
+ LOG.debug("LeaseServiceImpl, name: " + name + " " + getSelfUser() + " 续约锁成功, 租约到期时间为 "
+ + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(leaseDate));
+ } else {
+ iscontinue = false;
+ synchronized (leaseService) {
+ holdLockTasks.remove(name);
+ }
+ LOG.info("LeaseServiceImpl name: " + name + " " + getSelfUser() + " 续约锁失败，续锁程序会停止");
+ }
+ } catch (Exception e) {
+ iscontinue = false;
+ LOG.error(" LeaseServiceImpl name: " + name + " " + getSelfUser() + " 续约锁出现异常，续锁程序会停止", e);
+ }
+
+ }
+
+ }
+
+ /**
+ * 抢占锁的future，必须等锁超时才能继续获取锁
+ */
+ protected class HoldLockFunture implements Future<Boolean> {
+ private volatile boolean isDone = false;
+ private volatile Date date = null;
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ throw new RuntimeException("can not cancel");
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ if (date != null && System.currentTimeMillis() - date.getTime() >= 0) {
+ isDone = true;
+ return isDone;
+ }
+ return false;
+ }
+
+ @Override
+ public Boolean get() throws InterruptedException, ExecutionException {
+ while (isDone() == false) {
+ Thread.sleep(1000);
+ }
+ return true;
+ }
+
+ private long startTime = System.currentTimeMillis();
+
+ @Override
+ public Boolean get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+
+ throw new RuntimeException("can not support timeout ");
+ }
+
+ }
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/MockLeaseImpl.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/MockLeaseImpl.java
new file mode 100644
index 0000000..d952581
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/impl/MockLeaseImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.impl;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+
+/**
+ * 在内存和文件模式下使用，所有的申请都会返回true，主要用来做业务测试
+ */
+public class MockLeaseImpl implements ILeaseService {
+ @Override
+ public boolean hasLease(String name) {
+ return true;
+ }
+
+ @Override
+ public void startLeaseTask(String name) {
+
+ }
+
+ @Override
+ public void startLeaseTask(String name, ILeaseGetCallback callback) {
+ callback.callback(DateUtil.addMinute(new Date(), 1));
+ }
+
+ @Override
+ public void startLeaseTask(String name, int leaseTerm, ILeaseGetCallback callback) {
+
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName) {
+ return true;
+ }
+
+ @Override
+ public boolean lock(String name, String lockerName, int lockTimeSecond) {
+ return true;
+ }
+
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime) {
+ return true;
+ }
+
+ @Override
+ public boolean tryLocker(String name, String lockerName, long waitTime, int lockTimeSecond) {
+ return true;
+ }
+
+ @Override
+ public boolean unlock(String name, String lockerName) {
+ return true;
+ }
+
+ @Override
+ public boolean holdLock(String name, String lockerName, int lockTimeSecond) {
+ return true;
+ }
+
+ @Override
+ public boolean hasHoldLock(String name, String lockerName) {
+ return true;
+ }
+
+ @Override
+ public List<LeaseInfo> queryLockedInstanceByNamePrefix(String name, String lockerNamePrefix) {
+ return null;
+ }
+
+}
diff --git a/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/storages/DBLeaseStorage.java b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/storages/DBLeaseStorage.java
new file mode 100644
index 0000000..b99132d
--- /dev/null
+++ b/rocketmq-streams-lease/src/main/java/org/apache/rocketmq/streams/lease/service/storages/DBLeaseStorage.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease.service.storages;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.rocketmq.streams.common.utils.DateUtil;
+import org.apache.rocketmq.streams.common.utils.SQLUtil;
+import org.apache.rocketmq.streams.common.utils.StringUtil;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseStorage;
+
+public class DBLeaseStorage implements ILeaseStorage {
+ private static final Log LOG = LogFactory.getLog(DBLeaseStorage.class);
+ protected JDBCDriver jdbcDataSource;
+ private String url;
+ protected String userName;
+ protected String password;
+ protected String jdbc;
+
+ public DBLeaseStorage(String jdbc, String url, String userName, String password) {
+ this.jdbc = jdbc;
+ this.url = url;
+ this.userName = userName;
+ this.password = password;
+ jdbcDataSource = DriverBuilder.createDriver(jdbc, url, userName, password);
+ }
+
+ @Override
+ public boolean updateLeaseInfo(LeaseInfo leaseInfo) {
+ String sql = "UPDATE lease_info SET version=version+1,status=#{status},gmt_modified=now()";
+ String whereSQL = " WHERE id=#{id} and version=#{version}";
+
+ if (StringUtil.isNotEmpty(leaseInfo.getLeaseName())) {
+ sql += ",lease_name=#{leaseName}";
+ }
+ if (StringUtil.isNotEmpty(leaseInfo.getLeaseUserIp())) {
+ sql += ",lease_user_ip=#{leaseUserIp}";
+ }
+ if (leaseInfo.getLeaseEndDate() != null) {
+ sql += ",lease_end_time=#{leaseEndDate}";
+ }
+ sql += whereSQL;
+ sql = SQLUtil.parseIbatisSQL(leaseInfo, sql);
+ try {
+ int count = getOrCreateJDBCDataSource().update(sql);
+ boolean success = count > 0;
+ if (success) {
+ synchronized (this) {
+ leaseInfo.setVersion(leaseInfo.getVersion() + 1);
+ }
+ } else {
+ System.out.println(count);
+ }
+ return success;
+ } catch (Exception e) {
+ LOG.error("LeaseServiceImpl updateLeaseInfo excuteUpdate error", e);
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ @Override
+ public Integer countLeaseInfo(String leaseName) {
+ String sql = "SELECT count(*) as c FROM lease_info WHERE lease_name = '" + leaseName + "' and status = 1";
+ try {
+
+ List<Map<String, Object>> rows = getOrCreateJDBCDataSource().queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ Long value = (Long)rows.get(0).get("c");
+ return value.intValue();
+ } catch (Exception e) {
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ @Override
+ public LeaseInfo queryInValidateLease(String leaseName) {
+ String sql = "SELECT * FROM lease_info WHERE lease_name ='" + leaseName + "' and status=1 and lease_end_time<'" + DateUtil.getCurrentTimeString() + "'";
+ LOG.info("LeaseServiceImpl queryInValidateLease builder:" + sql);
+ return queryLease(leaseName, sql);
+ }
+
+ @Override
+ public LeaseInfo queryValidateLease(String leaseName) {
+ String sql = "SELECT * FROM lease_info WHERE lease_name ='" + leaseName + "' and status=1 and lease_end_time>now()";
+ return queryLease(leaseName, sql);
+ }
+
+ @Override
+ public List<LeaseInfo> queryValidateLeaseByNamePrefix(String namePrefix) {
+ String sql = "SELECT * FROM lease_info WHERE lease_name like '" + namePrefix + "%' and status=1 and lease_end_time>now()";
+ try {
+ List<LeaseInfo> leaseInfos = new ArrayList<>();
+ List<Map<String, Object>> rows = getOrCreateJDBCDataSource().queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ for (Map<String, Object> row : rows) {
+ LeaseInfo leaseInfo = convert(row);
+ leaseInfos.add(leaseInfo);
+ }
+
+ return leaseInfos;
+ } catch (Exception e) {
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ @Override
+ public void addLeaseInfo(LeaseInfo leaseInfo) {
+ String sql =
+ " REPLACE INTO lease_info(lease_name,lease_user_ip,lease_end_time,status,version,gmt_create,gmt_modified)"
+ + " VALUES (#{leaseName},#{leaseUserIp},#{leaseEndDate},#{status},#{version},now(),now())";
+ sql = SQLUtil.parseIbatisSQL(leaseInfo, sql);
+ try {
+
+ getOrCreateJDBCDataSource().execute(sql);
+ } catch (Exception e) {
+ LOG.error("LeaseServiceImpl execute sql error,sql:" + sql, e);
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ protected JDBCDriver getOrCreateJDBCDataSource() {
+ if (this.jdbcDataSource == null || !this.jdbcDataSource.isValidate()) {
+ synchronized (this) {
+ if (this.jdbcDataSource == null || !this.jdbcDataSource.isValidate()) {
+ this.jdbcDataSource =
+ DriverBuilder.createDriver(this.jdbc, this.url, this.userName, this.password);
+ }
+ }
+ }
+ return jdbcDataSource;
+ }
+
+ protected LeaseInfo queryLease(String name, String sql) {
+ try {
+ List<Map<String, Object>> rows = getOrCreateJDBCDataSource().queryForList(sql);
+ if (rows == null || rows.size() == 0) {
+ return null;
+ }
+ return convert(rows.get(0));
+ } catch (Exception e) {
+ throw new RuntimeException("execute sql error " + sql, e);
+ }
+ }
+
+ protected LeaseInfo convert(Map<String, Object> map) {
+ LeaseInfo leaseInfo = new LeaseInfo();
+ leaseInfo.setId(getMapLongValue("id", map));
+ leaseInfo.setCreateTime(getMapDateValue("gmt_create", map));
+ leaseInfo.setLeaseEndDate(getMapDateValue("lease_end_time", map));
+ leaseInfo.setLeaseName(getMapValue("lease_name", map, String.class));
+ leaseInfo.setLeaseUserIp(getMapValue("lease_user_ip", map, String.class));
+ Integer stauts = getMapValue("status", map, Integer.class);
+ if (stauts != null) {
+ leaseInfo.setStatus(stauts);
+ }
+ leaseInfo.setUpdateTime(getMapDateValue("gmt_modified", map));
+ Long version = getMapLongValue("version", map);
+ if (version != null) {
+ leaseInfo.setVersion(version);
+ }
+ return leaseInfo;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> T getMapValue(String fieldName, Map<String, Object> map, Class<T> integerClass) {
+ Object value = map.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ return (T)value;
+ }
+
+ private Long getMapLongValue(String fieldName, Map<String, Object> map) {
+ Object value = map.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Long) {
+ return (Long)value;
+ }
+ if (value instanceof BigInteger) {
+ return ((BigInteger)value).longValue();
+ }
+ return null;
+ }
+
+ private Date getMapDateValue(String fieldName, Map<String, Object> map) {
+ Object value = map.get(fieldName);
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date)value;
+ }
+ if (value instanceof String) {
+ return DateUtil.parseTime(((String)value));
+ }
+ return null;
+
+ }
+
+}
diff --git a/rocketmq-streams-lease/src/test/java/org/apache/rocketmq/streams/lease/LeaseComponentTest.java b/rocketmq-streams-lease/src/test/java/org/apache/rocketmq/streams/lease/LeaseComponentTest.java
new file mode 100644
index 0000000..6404e9f
--- /dev/null
+++ b/rocketmq-streams-lease/src/test/java/org/apache/rocketmq/streams/lease/LeaseComponentTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.streams.lease;
+
+import java.util.Date;
+
+import org.apache.rocketmq.streams.common.component.ComponentCreator;
+import org.apache.rocketmq.streams.common.configure.ConfigureFileKey;
+import org.apache.rocketmq.streams.db.driver.DriverBuilder;
+import org.apache.rocketmq.streams.db.driver.JDBCDriver;
+import org.apache.rocketmq.streams.lease.model.LeaseInfo;
+import org.apache.rocketmq.streams.lease.service.ILeaseGetCallback;
+import org.apache.rocketmq.streams.lease.service.ILeaseService;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class LeaseComponentTest {
+
+ private String URL = "";
+ protected String USER_NAME = "";
+ protected String PASSWORD = "";
+
+ public LeaseComponentTest() {
+
+ //正式使用时，在配置文件配置
+ ComponentCreator.getProperties().put(ConfigureFileKey.CONNECT_TYPE, "DB");
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_URL, URL);//数据库连接url
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_USERNAME, USER_NAME);//用户名
+ ComponentCreator.getProperties().put(ConfigureFileKey.JDBC_PASSWORD, PASSWORD);//password
+
+ JDBCDriver driver = DriverBuilder.createDriver();
+ driver.execute(LeaseInfo.createTableSQL());
+ }
+
+ @Test
+ public void testLease() throws InterruptedException {
+ String leaseName = "lease.test";
+ int leaseTime = 5;
+ LeaseComponent.getInstance().getService().startLeaseTask(leaseName, leaseTime, new ILeaseGetCallback() {
+ @Override
+ public void callback(Date nextLeaseDate) {
+ System.out.println("I get lease");
+ }
+ });
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ Thread.sleep(5000);
+ assertTrue(LeaseComponent.getInstance().getService().hasLease(leaseName));//会一直续约
+ }
+
+ @Test
+ public void testLock() throws InterruptedException {
+ String name = "dipper";
+ String lockName = "lease.test";
+ int leaseTime = 5;
+ boolean success = LeaseComponent.getInstance().getService().lock(name, lockName, leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(6000);
+ assertFalse(LeaseComponent.getInstance().getService().hasHoldLock(name, lockName));//超期释放
+ }
+
+ /**
+ * holdlock是一直持有锁，和租约的区别是，当释放锁后，无其他实例抢占
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testHoldLock() throws InterruptedException {
+ String name = "dipper";
+ String lockName = "lease.test";
+ int leaseTime = 6;
+ boolean success = LeaseComponent.getInstance().getService().holdLock(name, lockName, leaseTime);//锁定5秒钟
+ assertTrue(success);//获取锁
+ Thread.sleep(8000);
+ assertTrue(LeaseComponent.getInstance().getService().hasHoldLock(name, lockName));//会自动续约，不会释放，可以手动释放
+ LeaseComponent.getInstance().getService().unlock(name, lockName);
+ assertFalse(LeaseComponent.getInstance().getService().hasHoldLock(name, lockName));
+ }
+
+ @Test
+ public void testHoldLockContinue() throws InterruptedException {
+ String name = "dipper";
+ String lockName = "lease.test";
+ int leaseTime = 6;
+ boolean success = holdLock(name, lockName, leaseTime);//锁定5秒钟
+ while (true) {
+ Thread.sleep(1000);
+ System.out.println(holdLock(name, lockName, leaseTime));
+ }
+ }
+
+ protected boolean holdLock(String name, String lockName, int leaseTime) {
+ ILeaseService leaseService = LeaseComponent.getInstance().getService();
+ if (leaseService.hasHoldLock(name, lockName)) {
+ return true;
+ }
+
+ boolean success = leaseService.holdLock(name, lockName, leaseTime);
+ return success;
+ }
+
+}
diff --git a/rocketmq-streams-lease/src/test/resources/log4j.xml b/rocketmq-streams-lease/src/test/resources/log4j.xml
new file mode 100755
index 0000000..7812fe7
--- /dev/null
+++ b/rocketmq-streams-lease/src/test/resources/log4j.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "http://toolkit.alibaba-inc.com/dtd/log4j/log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="Console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} %l [%t] %-5p - %m%n%n"/>
+ </layout>
+ <filter class="org.apache.log4j.varia.LevelRangeFilter">
+ <param name="LevelMin" value="INFO"/>
+ <param name="LevelMax" value="ERROR"/>
+ </filter>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="Console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file