You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by du...@apache.org on 2021/11/30 02:17:34 UTC
[rocketmq-streams] 01/02: Merge pull request #87 from yuanxiaodong/window
This is an automated email from the ASF dual-hosted git repository.
duhengforever pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit e094ca0851064ea6b6a50d245047751c50ca3023
Merge: 32d41d3 9df6784
Author: YUDA <xs...@live.cn>
AuthorDate: Mon Oct 25 11:41:37 2021 +0800
Merge pull request #87 from yuanxiaodong/window
Window
README-chinese.md | 106 ++++++
build_without_test.sh | 2 +-
pom.xml | 19 +-
.../org/apache/rocketmq/streams/db/CycleSplit.java | 58 ++++
.../streams/db/DynamicMultipleDBSplit.java | 81 +++++
.../streams/db/sink/AbstractMultiTableSink.java | 44 ++-
.../apache/rocketmq/streams/db/sink/DBSink.java | 58 ++++
.../streams/db/sink/DynamicMultipleDBSink.java | 91 +++++
.../rocketmq/streams/db/sink/EnhanceDBSink.java | 371 +++++++++++++++++++++
.../streams/db/sink/sqltemplate/ISqlTemplate.java | 42 +--
.../MysqlInsertIgnoreIntoSqlTemplate.java | 24 +-
.../sqltemplate/MysqlInsertIntoSqlTemplate.java | 67 ++++
...MysqlInsertIntoWithDuplicateKeySqlTemplate.java | 55 +++
.../db/sink/sqltemplate/SqlTemplateFactory.java | 41 +++
.../streams/db/sink/db/DBWriteOnlyChannelTest.java | 10 +-
.../streams/db/sink/db/ISqlTemplateTest.java | 86 +++++
.../apache/rocketmq/streams/sink/RocketMQSink.java | 11 +-
.../pom.xml | 50 +--
.../streams/checkpoint/db/DBCheckPointStorage.java | 71 ++++
rocketmq-streams-clients/pom.xml | 6 +-
.../rocketmq/streams/client/DataStreamAction.java | 31 +-
.../streams/client/ScheduledStreamBuilder.java | 68 ++++
.../rocketmq/streams/client/ScheduledTask.java | 70 ++++
.../streams/client/source/DataStreamSource.java | 99 +++++-
.../client/strategy/LogFingerprintStrategy.java | 57 ++++
.../streams/client/transform/DataStream.java | 118 +++++--
.../streams/client/transform/OverWindowStream.java | 85 +++++
.../client/transform/window/WindowInfo.java | 2 +
.../apache/rocketmq/streams/client/DBSinkTest.java | 91 +++++
.../apache/rocketmq/streams/client/SourceTest.java | 8 +
.../client/windows/ShuffleOverWindowTest.java | 60 ++++
.../streams/common/cache/compress/BitSetCache.java | 77 +++--
.../common/cache/compress/ByteArrayValueKV.java | 26 +-
.../cache/compress/impl/FixedLenRowCacheKV.java | 2 +-
.../common/cache/compress/impl/ListValueKV.java | 2 +-
.../common/cache/compress/impl/LongValueKV.java | 2 +-
.../common/cache/compress/impl/StringValueKV.java | 6 +-
.../streams/common/channel/AbstractChannel.java | 17 +
.../impl/CollectionSink.java} | 48 +--
.../common/channel/impl/CollectionSinkBuilder.java | 51 +++
.../common/channel/impl/CollectionSource.java | 163 +++++++++
.../common/channel/impl/OutputPrintChannel.java | 11 +-
.../common/channel/impl/memory/MemoryChannel.java | 7 +
.../common/channel/impl/transit/TransitSink.java | 63 ++--
.../streams/common/channel/sink/AbstractSink.java | 35 +-
.../channel/sink/AbstractSupportShuffleSink.java | 6 +-
.../streams/common/channel/sink/ISink.java | 4 +
.../channel/sinkcache/impl/MessageCache.java | 8 +-
.../common/channel/source/AbstractSource.java | 105 ++++--
.../streams/common/channel/source/ISource.java | 19 +-
.../systemmsg/ChangeTableNameMessage.java} | 34 +-
.../checkpoint/AbstractCheckPointStorage.java | 192 +++++++++++
.../streams/common/checkpoint/CheckPoint.java | 98 ++++--
.../common/checkpoint/CheckPointManager.java | 261 ++++++---------
.../common/checkpoint/CheckPointMessage.java | 10 +
.../checkpoint/CheckPointStorageFactory.java | 84 +++++
.../ICheckPointStorage.java} | 26 +-
.../streams/common/checkpoint/SourceSnapShot.java | 77 +++++
.../streams/common/checkpoint/SourceState.java | 57 ++++
.../streams/common/configure/ConfigureFileKey.java | 8 +
.../streams/common/configure/StreamsConfigure.java | 59 ++++
.../streams/common/context/AbstractContext.java | 33 +-
.../IExpressionResultCache.java} | 9 +-
.../DateTimeDataType.java} | 35 +-
.../streams/common/datatype/HllDataType.java | 1 -
.../streams/common/functions/FlatMapFunction.java | 4 +-
.../streams/common/interfaces/IFilterService.java | 10 +
.../ILifeCycle.java} | 12 +-
.../streams/common/metadata/AbstractMetaData.java | 26 +-
.../streams/common/metadata/LogicMetaData.java | 57 ++++
.../streams/common/metadata/MetaDataUtils.java | 236 +++++++++++++
.../optimization/CalculationResultCache.java | 63 ----
.../common/optimization/FilterResultCache.java | 52 +++
.../common/optimization/HyperscanRegex.java | 52 +--
.../streams/common/optimization/LikeRegex.java | 68 +++-
.../common/optimization/LogFingerprintFilter.java | 152 ---------
.../common/optimization/OptimizationRegex.java | 6 +-
.../optimization/SQLLogFingerprintFilter.java | 53 ---
.../cachefilter/CacheFilterFactory.java | 64 ----
.../optimization/cachefilter/CacheFilterGroup.java | 81 -----
.../cachefilter/CacheFilterManager.java | 52 ---
.../optimization/fingerprint/FingerprintCache.java | 221 ++++++++++++
.../fingerprint/FingerprintMetric.java | 97 ++++++
.../optimization/fingerprint/PreFingerprint.java | 97 ++++++
.../AbstractMutilPipelineChainPipline.java | 4 +
.../streams/common/topology/ChainPipeline.java | 21 +-
.../streams/common/topology/ChainStage.java | 8 +-
.../common/topology/builder/PipelineBuilder.java | 3 +
.../common/topology/model/AbstractRule.java | 11 +
.../common/topology/model/AbstractStage.java | 176 ++++------
.../streams/common/topology/model/Pipeline.java | 42 +++
.../common/topology/stages/FilterChainStage.java | 205 +++++++-----
.../common/topology/stages/OutputChainStage.java | 1 +
.../topology/stages/SubPiplineChainStage.java | 98 +++---
.../common/topology/stages/udf/UDFChainStage.java | 42 +--
.../streams/common/utils/ContantsUtil.java | 5 +-
.../streams/common/utils/DataTypeUtil.java | 25 +-
.../rocketmq/streams/common/utils/MapKeyUtil.java | 26 +-
.../rocketmq/streams/common/utils/ReflectUtil.java | 3 +
.../rocketmq/streams/common/utils/SQLUtil.java | 171 ++++++++++
.../rocketmq/streams/common/utils/TraceUtil.java | 11 +-
rocketmq-streams-configurable/pom.xml | 1 +
.../src/main/resources/log4j.xml | 36 --
.../pom.xml | 14 +-
.../rocketmq/streams/connectors/IBounded.java | 12 +-
.../streams/connectors/IBoundedSource.java | 24 +-
.../streams/connectors/IBoundedSourceReader.java | 11 +-
.../connectors/balance/AbstractBalance.java | 204 +++++++++++
.../streams/connectors/balance/IBalanceTask.java | 11 +-
.../streams/connectors/balance/ISourceBalance.java | 50 +--
.../streams/connectors/balance/SplitChanged.java | 56 ++++
.../connectors/balance/impl/LeaseBalanceImpl.java | 141 ++++++++
.../streams/connectors/model/PullMessage.java | 50 +--
.../streams/connectors/model/ReaderStatus.java | 121 +++++++
.../streams/connectors/reader/DBScanReader.java | 270 +++++++++++++++
.../streams/connectors/reader/ISplitReader.java | 101 ++++++
.../connectors/reader/SplitCloseFuture.java | 83 +++++
.../connectors/source/AbstractPullSource.java | 271 +++++++++++++++
.../source/CycleDynamicMultipleDBScanSource.java | 213 ++++++++++++
.../source/DynamicMultipleDBScanSource.java | 192 +++++++++++
.../streams/connectors/source/IPullSource.java | 46 ++-
.../streams/connectors/source/SourceInstance.java | 31 +-
.../source/filter/AbstractPatternFilter.java | 21 +-
.../source/filter/BoundedPatternFilter.java | 56 ++++
.../source/filter/CyclePatternFilter.java | 173 ++++++++++
.../connectors/source/filter/CyclePeriod.java | 231 +++++++++++++
.../connectors/source/filter/CycleSchedule.java | 239 +++++++++++++
.../source/filter/CycleScheduleFilter.java | 18 +-
.../source/filter/DataFormatPatternFilter.java | 109 ++++++
.../connectors/source/filter/PatternFilter.java | 29 +-
.../rocketmq/streams/db/driver/orm/ORMUtil.java | 27 ++
.../src/main/resources/tables_mysql_innodb.sql | 16 +
.../rocketmq/streams/filter/FilterComponent.java | 5 +-
.../streams/filter/builder/ExpressionBuilder.java | 26 +-
.../streams/filter/builder/RuleBuilder.java | 25 ++
.../streams/filter/engine/IRuleEngine.java | 8 +-
.../filter/engine/impl/DefaultRuleEngine.java | 27 +-
.../filter/function/expression/InFunction.java | 4 +
.../filter/function/expression/IsNotNull.java | 4 +
.../streams/filter/function/expression/IsNull.java | 4 +-
.../filter/function/expression/LikeFunction.java | 8 -
.../filter/function/expression/RegexFunction.java | 20 +-
.../filter/function/script/CaseFunction.java | 2 +-
.../rocketmq/streams/filter/operator/Rule.java | 32 +-
.../operator/action/impl/CaseWhenAction.java | 46 +++
.../filter/operator/expression/Expression.java | 74 +++-
.../operator/expression/GroupExpression.java | 96 ++++--
.../expression/GroupExpressionManager.java | 16 +-
.../operator/expression/RelationExpression.java | 61 ++--
.../operator/expression/SimpleExpression.java | 57 ----
.../optimization/AbstractExpressionProxy.java | 70 ----
.../streams/filter/optimization/CaseProxy.java | 97 ------
.../optimization/EqualsExpressionOptimization.java | 39 ---
.../filter/optimization/EqualsExpressionProxy.java | 33 --
.../optimization/ExpressionProxyFactory.java | 61 ----
.../optimization/LikeExpressionOptimization.java | 39 ---
.../filter/optimization/LikeExpressionProxy.java | 33 --
.../optimization/OptimizationExpression.java | 71 ----
.../PiplineLogFingerprintAnalysis.java | 7 +-
.../optimization/RegexExpressionOptimization.java | 39 ---
.../casewhen/AbstractWhenExpression.java | 208 ++++++++++++
.../optimization/casewhen/CaseWhenBuilder.java | 234 +++++++++++++
.../optimization/casewhen/CaseWhenElement.java | 166 +++++++++
.../optimization/casewhen/GroupByVarCaseWhen.java | 123 +++++++
.../casewhen/MutilCaseWhenExpression.java | 39 +--
.../casewhen/SingleCaseWhenExpression.java | 84 +++++
.../optimization/dependency/DependencyTree.java | 13 +-
.../optimization/dependency/FilterTreeNode.java | 111 ++++++
.../optimization/dependency/PipelineTree.java | 57 ++++
.../optimization/dependency/ScriptDependent.java | 110 ++++++
.../optimization/dependency/ScriptTreeNode.java | 45 +++
.../dependency/SimplePipelineTree.java | 63 ++++
.../filter/optimization/dependency/TreeNode.java | 76 +++++
.../optimization/dependency/UnionTreeNode.java | 71 ++++
.../optimization/executor/AbstractExecutor.java | 29 +-
.../optimization/executor/GroupByVarExecutor.java | 212 ++++++++++++
.../optimization/executor/HyperscanExecutor.java | 141 ++++++++
.../result/GroupQuickFilterResult.java | 70 ++++
.../optimization/script/ScriptOptimization.java | 48 +++
.../streams/filter/service/IRuleEngineService.java | 9 -
.../filter/service/impl/RuleEngineServiceImpl.java | 23 +-
.../lease/service/impl/BasedLesaseImpl.java | 2 +-
.../rocketmq/streams/lease/LeaseComponentTest.java | 4 +-
rocketmq-streams-script/pom.xml | 2 +-
.../streams/script/context/FunctionContext.java | 12 +-
.../aggregation/FirstValueAccumulator.java | 78 +++++
.../function/impl/between/BetweenFunction.java | 74 ++++
.../function/impl/condition/IFScopeFunction.java | 40 +--
.../impl/date/CurrentTimestampFunction.java | 60 +++-
.../function/impl/date/DateFormatFunction.java | 17 +-
.../function/impl/date/DateUtileFunction.java | 17 +
.../function/impl/distinct/DistinctFunction.java | 68 ++++
.../function/impl/field/RemoveFieldFunction.java | 11 +
.../function/impl/flatmap/SplitArrayFunction.java | 7 +-
.../ItemFunction.java} | 39 ++-
.../function/impl/json/JsonValueFunction.java | 3 +
.../script/function/impl/math/BitFunction.java | 14 +-
.../function/impl/math/IsDecimalFunction.java | 83 +++++
.../script/function/impl/math/MathFunction.java | 21 ++
.../script/function/impl/string/RegexFunction.java | 11 +-
.../{ToLowerFunction.java => String2Map.java} | 32 +-
.../function/impl/string/ToLowerFunction.java | 10 +
.../script/function/impl/udtf/UDTFFunction.java | 2 +-
.../script/function/model/FunctionConfigure.java | 1 +
.../function/model/FunctionConfigureMap.java | 3 +
.../script/function/model/FunctionType.java | 21 +-
.../script/function/service/IFunctionService.java | 3 +
.../service/impl/DefaultFunctionServiceImpl.java | 23 ++
.../operator/expression/GroupScriptExpression.java | 82 ++++-
.../operator/expression/ScriptExpression.java | 29 +-
.../script/operator/impl/FunctionScript.java | 32 +-
.../performance/AbstractScriptProxy.java | 83 -----
.../performance/CaseScriptExpressionProxy.java | 70 ----
.../optimization/performance/EqualsProxy.java | 43 ---
.../performance/IScriptOptimization.java | 75 +++++
.../optimization/performance/RegexProxy.java | 43 ---
.../performance/ScriptExpressionGroupsProxy.java | 90 -----
.../performance/ScriptOptimization.java | 291 ----------------
.../performance/ScriptProxyFactory.java | 59 ----
.../performance/SimpleScriptExpressionProxy.java | 75 -----
.../script/service/impl/ScriptServiceImpl.java | 3 +
.../streams/script/utils/ExpressionUtil.java | 66 ++++
.../streams/script/utils/FunctionUtils.java | 8 +-
.../src/main/resources/log4j.xml | 36 --
.../component/ServiceLoaderComponent.properties | 18 -
rocketmq-streams-state/pom.xml | 3 +-
.../rocketmq/streams/state/AbstractState.java | 65 ++++
.../org/apache/rocketmq/streams/state/ISchama.java | 24 +-
.../streams/state/backend/IStateBackend.java | 151 +++++++++
.../rocketmq/streams/state/impl/MapState.java | 12 +-
.../rocketmq/streams/state/kv/TestLruState.java | 8 +-
.../streams/state/kv/TestRocksdbState.java | 10 +-
rocketmq-streams-window/pom.xml | 2 -
.../streams/window/fire/EventTimeManager.java | 14 +-
.../rocketmq/streams/window/model/FireMode.java | 16 +
.../streams/window/model/WindowInstance.java | 107 +++++-
.../window/operator/AbstractShuffleWindow.java | 7 +-
.../streams/window/operator/AbstractWindow.java | 179 ++++++----
.../streams/window/operator/impl/OrderBy.java | 40 ++-
.../streams/window/operator/impl/OverWindow.java | 20 +-
.../window/operator/impl/ShuffleOverWindow.java | 132 ++++++++
.../streams/window/operator/impl/TopNState.java | 262 +++++++++++++++
.../window/operator/impl/WindowOperator.java | 17 +-
.../streams/window/operator/join/JoinWindow.java | 2 +-
.../window/shuffle/AbstractSystemChannel.java | 2 +-
.../streams/window/shuffle/ShuffleChannel.java | 68 ++--
.../rocketmq/streams/window/sqlcache/SQLCache.java | 6 +-
.../streams/window/state/AbstractMapState.java | 5 +-
.../streams/window/state/impl/WindowValue.java | 14 +
.../streams/window/storage/IWindowStorage.java | 4 +-
.../streams/window/storage/WindowStorage.java | 4 +-
.../src/main/resources/dipper.properties | 21 --
.../rocketmq/streams/storage/RocksdbTest.java | 14 +-
.../streams/window/WindowInstanceTest.java | 5 +-
254 files changed, 11295 insertions(+), 3346 deletions(-)