You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by du...@apache.org on 2021/08/03 06:50:50 UTC
[rocketmq-streams] 22/35: modify README.md
This is an automated email from the ASF dual-hosted git repository.
duhengforever pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit d8216f0a8ee72e34a2a6ecd1d67c36c4bf5696dd
Merge: b013eb2 607418a
Author: xstorm1 <xs...@live.cn>
AuthorDate: Mon Aug 2 15:43:16 2021 +0800
modify README.md
.gitignore | 25 ++
README.md | 55 ++-
pom.xml | 410 +++++++++++++++++
rocketmq-streams-clients/pom.xml | 46 ++
.../rocketmq/streams/client/DataStreamAction.java | 101 +++++
.../rocketmq/streams/client/StreamBuilder.java | 28 ++
.../streams/client/source/DataStreamSource.java | 76 ++++
.../client/strategy/CheckpointStrategy.java | 69 +++
.../streams/client/strategy/StateStrategy.java | 37 ++
.../rocketmq/streams/client/strategy/Strategy.java | 25 ++
.../streams/client/transform/DataStream.java | 437 ++++++++++++++++++
.../streams/client/transform/JoinStream.java | 212 +++++++++
.../streams/client/transform/SplitStream.java | 61 +++
.../streams/client/transform/WindowStream.java | 210 +++++++++
.../client/transform/window/HoppingWindow.java | 32 ++
.../client/transform/window/SessionWindow.java | 32 ++
.../streams/client/transform/window/Time.java | 45 ++
.../client/transform/window/TumblingWindow.java | 33 ++
.../client/transform/window/WindowInfo.java | 83 ++++
.../rocketmq/streams/client/DBDriverTest.java | 75 ++++
.../rocketmq/streams/client/DataStreamTest.java | 107 +++++
.../apache/rocketmq/streams/client/FilterTest.java | 49 +++
.../apache/rocketmq/streams/client/JoinTest.java | 89 ++++
.../apache/rocketmq/streams/client/LeaseTest.java | 98 +++++
.../rocketmq/streams/client/ORMUtilTest.java | 172 ++++++++
.../apache/rocketmq/streams/client/SplitTest.java | 86 ++++
.../apache/rocketmq/streams/client/UnionTest.java | 82 ++++
.../apache/rocketmq/streams/client/WindowTest.java | 86 ++++
.../client/windows/AbstractWindowFireModeTest.java | 189 ++++++++
.../streams/client/windows/WindowFromFileTest.java | 158 +++++++
.../streams/client/windows/WindowFromMetaq.java | 47 ++
.../client/windows/WindowHighAvailabilityTest.java | 131 ++++++
.../src/test/resources/log4j.xml | 36 ++
rocketmq-streams-db-operator/pom.xml | 34 ++
.../rocketmq-streams-db-operator.iml | 16 +
.../streams/db/configuable/DBConfigureService.java | 282 ++++++++++++
.../DBSupportParentConfigureService.java | 37 ++
.../rocketmq/streams/db/driver/DriverBuilder.java | 111 +++++
.../rocketmq/streams/db/driver/IDriverBudiler.java | 36 ++
.../rocketmq/streams/db/driver/JDBCDriver.java | 277 ++++++++++++
.../db/driver/batchloader/BatchRowLoader.java | 179 ++++++++
.../db/driver/batchloader/IRowOperator.java | 33 ++
.../rocketmq/streams/db/driver/orm/ORMUtil.java | 490 +++++++++++++++++++++
.../rocketmq/streams/db/operator/SQLOperator.java | 178 ++++++++
.../org/apache/rocketmq/streams/db/Person.java | 110 +++++
.../DBSupportParentConfigureServiceTest.java | 74 ++++
.../streams/db/driver/orm/ORMUtilTest.java | 86 ++++
rocketmq-streams-dim/pom.xml | 47 ++
.../apache/rocketmq/streams/dim/DimComponent.java | 63 +++
.../rocketmq/streams/dim/builder/DimBuilder.java | 94 ++++
.../function/expression/InExpressionResource.java | 80 ++++
.../expression/NotInExpressionResource.java | 45 ++
.../dim/function/script/IntelligenceFunction.java | 81 ++++
.../script/IntelligenceNameListFunction.java | 24 +
.../dim/function/script/NameListFunction.java | 203 +++++++++
.../rocketmq/streams/dim/index/DimIndex.java | 319 ++++++++++++++
.../rocketmq/streams/dim/index/IndexExecutor.java | 258 +++++++++++
.../intelligence/AbstractIntelligenceCache.java | 395 +++++++++++++++++
.../dim/intelligence/AccountIntelligenceCache.java | 77 ++++
.../dim/intelligence/DomainIntelligenceCache.java | 83 ++++
.../dim/intelligence/IPIntelligenceCache.java | 108 +++++
.../dim/intelligence/URLIntelligenceCache.java | 80 ++++
.../rocketmq/streams/dim/model/AbstractDim.java | 312 +++++++++++++
.../streams/dim/model/BooleanFieldDBDim.java | 55 +++
.../apache/rocketmq/streams/dim/model/DBDim.java | 140 ++++++
.../rocketmq/streams/dim/service/IDimService.java | 65 +++
.../streams/dim/service/impl/DimServiceImpl.java | 92 ++++
.../com/aliyun/service/ConfigureLoaderTest.java | 37 ++
.../com/aliyun/service/ExpressionExecutorTest.java | 80 ++++
.../java/com/aliyun/service/JsonParserTest.java | 40 ++
.../com/aliyun/service/NameListFunctionTest.java | 90 ++++
.../java/com/aliyun/service/TableCompressTest.java | 26 ++
rocketmq-streams-lease/pom.xml | 25 ++
.../rocketmq/streams/lease/LeaseComponent.java | 103 +++++
.../rocketmq/streams/lease/model/LeaseInfo.java | 127 ++++++
.../streams/lease/service/ILeaseGetCallback.java | 30 ++
.../streams/lease/service/ILeaseService.java | 136 ++++++
.../streams/lease/service/ILeaseStorage.java | 73 +++
.../streams/lease/service/ILeaseStorasge.java | 63 +++
.../lease/service/impl/BasedLesaseImpl.java | 404 +++++++++++++++++
.../lease/service/impl/LeaseServiceImpl.java | 275 ++++++++++++
.../streams/lease/service/impl/MockLeaseImpl.java | 95 ++++
.../lease/service/storages/DBLeaseStorage.java | 229 ++++++++++
.../rocketmq/streams/lease/LeaseComponentTest.java | 119 +++++
.../src/test/resources/log4j.xml | 20 +
rocketmq-streams-transport-minio/pom.xml | 25 ++
.../rocketmq-streams-transport-minio.iml | 17 +
.../transport/minio/MinioFileTransport.java | 141 ++++++
.../yundun/dipper/configurable/DataTpyeTest.java | 70 +++
.../streams/configuable/model/DataTpyeTest.java | 68 +++
.../rocketmq/streams/configuable/model/Person.java | 97 ++++
.../streams/configurable/model/Person.java | 97 ++++
.../component/ConfigurableComponent.properties | 7 +
.../src/test/resources/log4j.xml | 20 +
.../src/test/resources/pro-function.txt | 11 +
.../src/test/resources/python_script.py | 22 +
96 files changed, 10532 insertions(+), 1 deletion(-)
diff --cc README.md
index 678bd5a,92a9b2f..d2ff23e
--- a/README.md
+++ b/README.md
@@@ -9,17 -8,15 +8,29 @@@
## DataStream Example
```java
++<<<<<<< HEAD
+ import org.apache.rocketmq.streams.client.transform.DataStream;
+
+
+
+ DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+
+ source
+ .fromFile("/Users/junjie.cheng/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
++=======
+ import org.apache.rocketmq.streams.client.transform.DataStream;
+
+ DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline");
+
+ source
+ .fromFile("/Users/junjie.cheng/text.txt",false)
+ .map(message->message)
+ .toPrint(1)
+ .start();
++>>>>>>> main
```
## Maven Repository
@@@ -47,6 -44,7 +58,10 @@@ StreamBuilder 用于构建流任务的�
## DataStream API
### Source
++<<<<<<< HEAD
++=======
+
++>>>>>>> main
DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据；
+ ```fromFile``` 从文件中读取数据， 该方法包含俩个参数
@@@ -60,13 -58,14 +75,24 @@@
+ ```isJson``` 是否json格式，非必填参数
+ ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数
++<<<<<<< HEAD
+
++ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
+
+### transform
+transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
+
+#### DataStream
++=======
+ + ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源
+
+ ### transform
+
+ transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类；
+
+ #### DataStream
+
++>>>>>>> main
DataStream实现了一系列常见的流计算算子
+ ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream
@@@ -80,30 -79,29 +106,57 @@@
+ ```toFile``` 将结果保存为文件，生成一个新的DataStreamAction实例
+ ```toDB``` 将结果保存到数据库
+ ```toRocketmq``` 将结果输出到rocketmq
++<<<<<<< HEAD
++ ```toSls``` 将结果输出到sls
++ ```to``` 将结果经过自定义的ISink接口输出到指定的存储
++ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
+ + ```count``` 在窗口内计数
+ + ```min``` 获取窗口内统计值的最小值
+ + ```max``` 获取窗口内统计值得最大值
+ + ```avg``` 获取窗口内统计值的平均值
+ + ```sum``` 获取窗口内统计值的加和值
+ + ```reduce``` 在窗口内进行自定义的汇总运算
++=======
+ + ```to``` 将结果经过自定义的ISink接口输出到指定的存储
+ + ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个
+ + ```count``` 在窗口内计数
+ + ```min``` 获取窗口内统计值的最小值
+ + ```max``` 获取窗口内统计值得最大值
+ + ```avg``` 获取窗口内统计值的平均值
+ + ```sum``` 获取窗口内统计值的加和值
+ + ```reduce``` 在窗口内进行自定义的汇总运算
++>>>>>>> main
+ ```join``` 根据条件将将俩个流进行关联， 合并为一个大流进行相关的运算
+ ```union``` 将俩个流进行合并
+ ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算
+ ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等
++<<<<<<< HEAD
+
+# Strategy
+策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
+
+```java
+ //指定checkpoint的存储策略
+ source
+ .fromRocketmq("TSG_META_INFO", "")
+ .map(message -> message + "--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX", "", "", 0L))
+ .start();
+```
++=======
+ # Strategy
+
+ 策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型；
+
+ ```java
+ //指定checkpoint的存储策略
+ source
+ .fromRocketmq("TSG_META_INFO","")
+ .map(message->message+"--")
+ .toPrint(1)
+ .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L))
+ .start();
-```
++```
++>>>>>>> main