You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2022/05/16 12:54:52 UTC
[flink] branch release-1.15 updated (20768b2f53f -> 5d5d4aee507)
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a change to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
from 20768b2f53f [FLINK-27255] [flink-avro] flink-avro does not support ser/de of large avro schema (#19645)
new 033273a658a [FLINK-26553][build] Add scalafmt for formatting the Scala codebase
new 01ca3f8939c [FLINK-27232][build] Explicitly configure scalafmt in each module
new 09293d618bb [FLINK-27317][build] Use absolute paths to .scalafmt.conf
new bea6bafac98 [FLINK-27317][build] Don't fork build for attaching sources
new 5d5d4aee507 [FLINK-27534][build] Format code with Spotless/scalafmt
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.editorconfig | 4 -
.scalafmt.conf | 70 +
.../flink-hadoop-compatibility/pom.xml | 25 +-
.../scala/hadoop/mapred/HadoopInputFormat.scala | 1 +
.../scala/hadoop/mapred/HadoopOutputFormat.scala | 1 +
.../scala/hadoop/mapreduce/HadoopInputFormat.scala | 2 +-
.../hadoop/mapreduce/HadoopOutputFormat.scala | 2 +-
.../hadoopcompatibility/scala/HadoopInputs.scala | 78 +-
.../scala/ScalaWritableTypeInfoTest.scala | 1 +
.../scala/WordCountMapredITCase.scala | 23 +-
.../scala/WordCountMapreduceITCase.scala | 29 +-
flink-connectors/flink-hcatalog/pom.xml | 26 +-
.../flink/hcatalog/scala/HCatInputFormat.scala | 444 ++--
.../flink-end-to-end-tests-scala/pom.xml | 27 +-
.../org/apache/flink/tests/scala/ScalaJob.scala | 16 +-
.../flink-quickstart-test/pom.xml | 21 +
.../test/Elasticsearch7SinkExample.scala | 39 +-
.../test-scripts/test_kubernetes_itcases.sh | 4 +-
.../test-scripts/test_streaming_gcp_pubsub.sh | 5 +-
flink-examples/flink-examples-batch/pom.xml | 25 +-
.../flink/examples/scala/clustering/KMeans.scala | 89 +-
.../examples/scala/graph/ConnectedComponents.scala | 81 +-
.../flink/examples/scala/graph/DeltaPageRank.scala | 104 +-
.../flink/examples/scala/graph/EnumTriangles.scala | 85 +-
.../flink/examples/scala/graph/PageRankBasic.scala | 72 +-
.../scala/graph/TransitiveClosureNaive.scala | 67 +-
.../flink/examples/scala/misc/PiEstimation.scala | 15 +-
.../examples/scala/relational/TPCHQuery10.scala | 176 +-
.../examples/scala/relational/TPCHQuery3.scala | 129 +-
.../examples/scala/relational/WebLogAnalysis.scala | 69 +-
.../flink/examples/scala/wordcount/WordCount.scala | 13 +-
flink-examples/flink-examples-streaming/pom.xml | 25 +-
.../scala/examples/async/AsyncClient.scala | 5 +-
.../scala/examples/async/AsyncIOExample.scala | 5 +-
.../scala/examples/iteration/IterateExample.scala | 72 +-
.../streaming/scala/examples/join/WindowJoin.scala | 22 +-
.../scala/examples/join/WindowJoinSampleData.scala | 31 +-
.../examples/socket/SocketWindowWordCount.scala | 31 +-
.../GroupedProcessingTimeWindowExample.scala | 11 +-
.../examples/windowing/SessionWindowing.scala | 39 +-
.../examples/windowing/TopSpeedWindowing.scala | 54 +-
.../scala/examples/windowing/WindowWordCount.scala | 44 +-
.../scala/examples/windowing/util/CarSource.scala | 9 +-
.../scala/examples/wordcount/WordCount.scala | 56 +-
.../scala/examples/wordcount/util/CLI.scala | 1 -
.../scala/examples/StreamingExamplesITCase.scala | 98 +-
flink-examples/flink-examples-table/pom.xml | 16 +
.../scala/basics/GettingStartedExample.scala | 99 +-
.../examples/scala/basics/StreamSQLExample.scala | 21 +-
.../examples/scala/basics/StreamTableExample.scala | 29 +-
.../scala/basics/WordCountSQLExample.scala | 25 +-
flink-libraries/flink-cep-scala/pom.xml | 26 +-
.../scala/org/apache/flink/cep/scala/CEP.scala | 59 +-
.../org/apache/flink/cep/scala/PatternStream.scala | 590 ++---
.../flink/cep/scala/conditions/Context.scala | 16 +-
.../scala/org/apache/flink/cep/scala/package.scala | 24 +-
.../flink/cep/scala/pattern/GroupPattern.scala | 47 +-
.../apache/flink/cep/scala/pattern/Pattern.scala | 627 +++---
.../apache/flink/cep/scala/pattern/package.scala | 36 +-
.../cep/scala/CEPScalaAPICompletenessTest.scala | 13 +-
...ernStreamScalaJavaAPIInteroperabilityTest.scala | 64 +-
.../pattern/PatternScalaAPICompletenessTest.scala | 9 +-
.../flink/cep/scala/pattern/PatternTest.scala | 111 +-
flink-libraries/flink-gelly-examples/pom.xml | 26 +-
.../graph/scala/examples/ConnectedComponents.scala | 101 +-
.../examples/GSASingleSourceShortestPaths.scala | 61 +-
.../scala/examples/SingleSourceShortestPaths.scala | 71 +-
flink-libraries/flink-gelly-scala/pom.xml | 25 +-
.../apache/flink/graph/scala/EdgesFunction.scala | 9 +-
.../graph/scala/EdgesFunctionWithVertexValue.scala | 11 +-
.../scala/org/apache/flink/graph/scala/Graph.scala | 1430 ++++++------
.../flink/graph/scala/NeighborsFunction.scala | 14 +-
.../scala/NeighborsFunctionWithVertexValue.scala | 23 +-
.../org/apache/flink/graph/scala/package.scala | 10 +-
.../flink/graph/scala/utils/EdgeToTuple3Map.scala | 1 -
.../graph/scala/utils/Tuple2ToVertexMap.scala | 1 -
.../flink/graph/scala/utils/Tuple3ToEdgeMap.scala | 1 -
.../graph/scala/utils/VertexToTuple2Map.scala | 1 -
.../scala/test/GellyScalaAPICompletenessTest.scala | 11 +-
.../flink/graph/scala/test/TestGraphUtils.scala | 53 +-
.../scala/test/operations/DegreesITCase.scala | 30 +-
.../operations/GraphCreationWithCsvITCase.scala | 119 +-
.../test/operations/GraphMutationsITCase.scala | 173 +-
.../test/operations/GraphOperationsITCase.scala | 217 +-
.../test/operations/JoinWithEdgesITCase.scala | 92 +-
.../test/operations/JoinWithVerticesITCase.scala | 31 +-
.../scala/test/operations/MapEdgesITCase.scala | 26 +-
.../scala/test/operations/MapVerticesITCase.scala | 23 +-
.../operations/ReduceOnEdgesMethodsITCase.scala | 88 +-
.../operations/ReduceOnNeighborMethodsITCase.scala | 67 +-
flink-scala/pom.xml | 26 +-
.../apache/flink/api/scala/AggregateDataSet.scala | 34 +-
.../apache/flink/api/scala/ClosureCleaner.scala | 201 +-
.../apache/flink/api/scala/CoGroupDataSet.scala | 93 +-
.../org/apache/flink/api/scala/CrossDataSet.scala | 25 +-
.../scala/org/apache/flink/api/scala/DataSet.scala | 1267 +++++------
.../flink/api/scala/ExecutionEnvironment.scala | 495 ++--
.../apache/flink/api/scala/GroupedDataSet.scala | 311 ++-
.../flink/api/scala/PartitionSortedDataSet.scala | 24 +-
.../flink/api/scala/SelectByMaxFunction.scala | 35 +-
.../flink/api/scala/SelectByMinFunction.scala | 31 +-
.../api/scala/UnfinishedCoGroupOperation.scala | 33 +-
.../api/scala/codegen/MacroContextHolder.scala | 9 +-
.../apache/flink/api/scala/codegen/TreeGen.scala | 381 ++--
.../flink/api/scala/codegen/TypeAnalyzer.scala | 214 +-
.../flink/api/scala/codegen/TypeDescriptors.scala | 34 +-
.../api/scala/codegen/TypeInformationGen.scala | 81 +-
.../acceptPartialFunctions/OnCoGroupDataSet.scala | 39 +-
.../acceptPartialFunctions/OnCrossDataSet.scala | 34 +-
.../impl/acceptPartialFunctions/OnDataSet.scala | 127 +-
.../acceptPartialFunctions/OnGroupedDataSet.scala | 86 +-
.../OnHalfUnfinishedKeyPairOperation.scala | 38 +-
.../OnJoinFunctionAssigner.scala | 34 +-
.../OnUnfinishedKeyPairOperation.scala | 39 +-
.../flink/api/scala/extensions/package.scala | 75 +-
.../org/apache/flink/api/scala/joinDataSet.scala | 108 +-
.../flink/api/scala/metrics/ScalaGauge.scala | 5 +-
.../scala/org/apache/flink/api/scala/package.scala | 58 +-
.../api/scala/typeutils/CaseClassComparator.scala | 25 +-
.../api/scala/typeutils/CaseClassSerializer.scala | 19 +-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 91 +-
.../api/scala/typeutils/EitherSerializer.scala | 8 +-
.../flink/api/scala/typeutils/EitherTypeInfo.scala | 6 +-
.../api/scala/typeutils/EnumValueComparator.scala | 6 +-
.../api/scala/typeutils/EnumValueSerializer.scala | 27 +-
.../api/scala/typeutils/EnumValueTypeInfo.scala | 14 +-
.../api/scala/typeutils/NothingSerializer.scala | 11 +-
.../api/scala/typeutils/OptionSerializer.scala | 15 +-
.../api/scala/typeutils/OptionTypeComparator.scala | 16 +-
.../flink/api/scala/typeutils/OptionTypeInfo.scala | 10 +-
.../scala/typeutils/ScalaCaseClassSerializer.scala | 23 +-
.../typeutils/ScalaEnumSerializerSnapshot.scala | 13 +-
.../api/scala/typeutils/ScalaNothingTypeInfo.scala | 2 +-
.../scala/typeutils/TraversableSerializer.scala | 55 +-
.../api/scala/typeutils/TraversableTypeInfo.scala | 6 +-
.../flink/api/scala/typeutils/TrySerializer.scala | 21 +-
.../flink/api/scala/typeutils/TryTypeInfo.scala | 10 +-
.../flink/api/scala/typeutils/TypeUtils.scala | 4 +-
.../apache/flink/api/scala/typeutils/Types.scala | 477 ++--
.../flink/api/scala/typeutils/UnitSerializer.scala | 15 +-
.../flink/api/scala/typeutils/UnitTypeInfo.scala | 2 +-
.../api/scala/unfinishedKeyPairOperation.scala | 98 +-
.../org/apache/flink/api/scala/utils/package.scala | 117 +-
.../runtime/types/FlinkScalaKryoInstantiator.scala | 30 +-
.../types/JavaIterableWrapperSerializer.scala | 10 +-
.../api/scala/DeltaIterationSanityCheckTest.scala | 94 +-
.../apache/flink/api/scala/MaxByOperatorTest.scala | 54 +-
.../apache/flink/api/scala/MinByOperatorTest.scala | 55 +-
.../flink/api/scala/SelectByFunctionTest.scala | 250 +-
.../scala/extensions/base/AcceptPFTestBase.scala | 7 +-
.../api/scala/extensions/data/KeyValuePair.scala | 14 +-
.../OnCoGroupDataSetTest.scala | 31 +-
.../OnCrossDataSetTest.scala | 11 +-
.../acceptPartialFunctions/OnDataSetTest.scala | 91 +-
.../OnGroupedDataSetTest.scala | 37 +-
.../OnHalfUnfinishedKeyPairOperationTest.scala | 90 +-
.../OnJoinFunctionAssignerTest.scala | 121 +-
.../OnUnfinishedKeyPairOperationTest.scala | 95 +-
.../SemanticPropertiesTranslationTest.scala | 65 +-
.../api/scala/io/CollectionInputFormatTest.scala | 79 +-
.../flink/api/scala/io/CsvInputFormatTest.scala | 92 +-
.../flink/api/scala/metrics/ScalaGaugeTest.scala | 1 -
.../scala/runtime/EnumValueComparatorTest.scala | 1 -
.../scala/runtime/GenericPairComparatorTest.scala | 17 +-
.../runtime/KryoGenericTypeSerializerTest.scala | 50 +-
.../runtime/ScalaSpecialTypesSerializerTest.scala | 44 +-
.../scala/runtime/TraversableSerializerTest.scala | 45 +-
.../scala/runtime/TupleComparatorILD2Test.scala | 5 +-
.../scala/runtime/TupleComparatorILD3Test.scala | 2 -
.../scala/runtime/TupleComparatorILDC3Test.scala | 4 +-
.../scala/runtime/TupleComparatorILDX1Test.scala | 6 +-
.../scala/runtime/TupleComparatorILDXC2Test.scala | 6 +-
.../scala/runtime/TupleComparatorISD1Test.scala | 3 +-
.../scala/runtime/TupleComparatorISD2Test.scala | 1 -
.../scala/runtime/TupleComparatorISD3Test.scala | 3 +-
.../runtime/TupleSerializerCompatibilityTest.scala | 17 +-
...TupleSerializerCompatibilityTestGenerator.scala | 29 +-
.../api/scala/runtime/TupleSerializerTest.scala | 113 +-
.../runtime/TupleSerializerTestInstance.scala | 22 +-
.../tuple/base/PairComparatorTestBase.scala | 13 +-
.../tuple/base/TupleComparatorTestBase.scala | 6 +-
.../api/scala/types/TypeInformationGenTest.scala | 180 +-
.../scala/typeutils/CaseClassTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/EitherSerializerTest.scala | 11 +-
.../api/scala/typeutils/EitherTypeInfoTest.scala | 9 +-
.../EnumValueSerializerCompatibilityTest.scala | 57 +-
.../scala/typeutils/EnumValueSerializerTest.scala | 10 +-
.../typeutils/EnumValueSerializerUpgradeTest.scala | 41 +-
.../scala/typeutils/EnumValueTypeInfoTest.scala | 9 +-
.../scala/typeutils/InstantiationUtilTest.scala | 17 +-
.../api/scala/typeutils/OptionTypeInfoTest.scala | 6 +-
.../ScalaCaseClassSerializerReflectionTest.scala | 10 +-
.../typeutils/ScalaCaseClassSerializerTest.scala | 8 +-
.../ScalaCaseClassSerializerUpgradeTest.scala | 39 +-
.../scala/typeutils/ScalaNothingTypeInfoTest.scala | 5 +-
.../TraversableSerializerUpgradeTest.scala | 223 +-
.../scala/typeutils/TraversableTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/TryTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/TypeExtractionTest.scala | 7 +-
.../api/scala/typeutils/TypeInfoFactoryTest.scala | 17 +-
.../api/scala/typeutils/UnitTypeInfoTest.scala | 5 +-
flink-streaming-scala/pom.xml | 26 +-
.../streaming/api/scala/AllWindowedStream.scala | 398 ++--
.../streaming/api/scala/AsyncDataStream.scala | 366 +--
.../api/scala/BroadcastConnectedStream.scala | 60 +-
.../streaming/api/scala/CloseableIterator.scala | 17 +-
.../streaming/api/scala/CoGroupedStreams.scala | 104 +-
.../streaming/api/scala/ConnectedStreams.scala | 266 ++-
.../flink/streaming/api/scala/DataStream.scala | 899 ++++----
.../streaming/api/scala/DataStreamUtils.scala | 58 +-
.../flink/streaming/api/scala/JoinedStreams.scala | 127 +-
.../flink/streaming/api/scala/KeyedStream.scala | 532 ++---
.../flink/streaming/api/scala/OutputTag.scala | 11 +-
.../api/scala/StreamExecutionEnvironment.scala | 1000 ++++----
.../flink/streaming/api/scala/WindowedStream.scala | 401 ++--
.../streaming/api/scala/async/AsyncFunction.scala | 56 +-
.../api/scala/async/JavaResultFutureWrapper.scala | 17 +-
.../streaming/api/scala/async/ResultFuture.scala | 40 +-
.../api/scala/async/RichAsyncFunction.scala | 31 +-
.../async/ScalaRichAsyncFunctionWrapper.scala | 15 +-
.../acceptPartialFunctions/OnConnectedStream.scala | 89 +-
.../impl/acceptPartialFunctions/OnDataStream.scala | 75 +-
.../acceptPartialFunctions/OnJoinedStream.scala | 39 +-
.../acceptPartialFunctions/OnKeyedStream.scala | 29 +-
.../acceptPartialFunctions/OnWindowedStream.scala | 49 +-
.../streaming/api/scala/extensions/package.scala | 77 +-
.../api/scala/function/AllWindowFunction.scala | 28 +-
.../scala/function/ProcessAllWindowFunction.scala | 75 +-
.../api/scala/function/ProcessWindowFunction.scala | 88 +-
.../api/scala/function/RichAllWindowFunction.scala | 14 +-
.../function/RichProcessAllWindowFunction.scala | 22 +-
.../scala/function/RichProcessWindowFunction.scala | 25 +-
.../api/scala/function/RichWindowFunction.scala | 21 +-
.../api/scala/function/StatefulFunction.scala | 12 +-
.../api/scala/function/WindowFunction.scala | 40 +-
.../function/util/ScalaAllWindowFunction.scala | 11 +-
.../util/ScalaAllWindowFunctionWrapper.scala | 8 +-
.../util/ScalaProcessWindowFunctionWrapper.scala | 44 +-
.../scala/function/util/ScalaReduceFunction.scala | 9 +-
.../scala/function/util/ScalaWindowFunction.scala | 11 +-
.../function/util/ScalaWindowFunctionWrapper.scala | 10 +-
.../apache/flink/streaming/api/scala/package.scala | 47 +-
.../api/scala/AllWindowTranslationTest.scala | 343 ++-
.../api/scala/AsyncDataStreamITCase.scala | 45 +-
.../streaming/api/scala/BroadcastStateITCase.scala | 17 +-
.../api/scala/CaseClassFieldAccessorTest.scala | 2 +-
.../streaming/api/scala/CoGroupJoinITCase.scala | 201 +-
.../streaming/api/scala/CoGroupedStreamsTest.scala | 9 +-
.../flink/streaming/api/scala/DataStreamTest.scala | 184 +-
.../streaming/api/scala/IntervalJoinITCase.scala | 22 +-
.../streaming/api/scala/JoinedStreamsTest.scala | 9 +-
.../api/scala/OutputFormatTestPrograms.scala | 40 +-
.../ReinterpretDataStreamAsKeyedStreamITCase.scala | 8 +-
.../streaming/api/scala/SideOutputITCase.scala | 124 +-
.../streaming/api/scala/SlotAllocationTest.scala | 38 +-
.../streaming/api/scala/StateTestPrograms.scala | 85 +-
.../api/scala/StreamExecutionEnvironmentTest.scala | 14 +-
.../api/scala/StreamingOperatorsITCase.scala | 9 +-
.../scala/StreamingScalaAPICompletenessTest.scala | 42 +-
.../api/scala/TimeWindowTranslationTest.scala | 21 +-
.../streaming/api/scala/WindowFunctionITCase.scala | 196 +-
.../streaming/api/scala/WindowReduceITCase.scala | 276 ++-
.../api/scala/WindowTranslationTest.scala | 358 ++-
.../scala/extensions/base/AcceptPFTestBase.scala | 16 +-
.../api/scala/extensions/data/KeyValuePair.scala | 14 +-
.../OnConnectedDataStreamTest.scala | 71 +-
.../acceptPartialFunctions/OnDataStreamTest.scala | 57 +-
.../OnJoinedStreamTest.scala | 45 +-
.../OnKeyedDataStreamTest.scala | 11 +-
.../OnWindowedStreamTest.scala | 11 +-
.../sink/filesystem/StreamingFileSinkTest.scala | 22 +-
.../CheckingIdentityRichAllWindowFunction.scala | 10 +-
...ckingIdentityRichProcessAllWindowFunction.scala | 2 -
...CheckingIdentityRichProcessWindowFunction.scala | 2 -
.../CheckingIdentityRichWindowFunction.scala | 13 +-
flink-table/flink-table-api-scala-bridge/pom.xml | 25 +-
.../api/bridge/scala/DataStreamConversions.scala | 115 +-
.../api/bridge/scala/StreamStatementSet.scala | 4 +-
.../api/bridge/scala/StreamTableEnvironment.scala | 1118 ++++-----
.../table/api/bridge/scala/TableConversions.scala | 121 +-
.../scala/internal/StreamStatementSetImpl.scala | 16 +-
.../internal/StreamTableEnvironmentImpl.scala | 73 +-
.../flink/table/api/bridge/scala/package.scala | 14 +-
.../internal/StreamTableEnvironmentImplTest.scala | 29 +-
.../types/TypeInfoDataTypeConverterScalaTest.scala | 14 +-
flink-table/flink-table-api-scala/pom.xml | 25 +-
.../table/api/ImplicitExpressionConversions.scala | 474 ++--
.../table/api/ImplicitExpressionOperations.scala | 169 +-
.../scala/org/apache/flink/table/api/package.scala | 41 +-
.../api/ExpressionsConsistencyCheckTest.scala | 51 +-
.../flink/table/api/ImplicitConversionsTest.scala | 15 +-
.../expressions/ObjectToExpressionScalaTest.scala | 5 +-
.../extraction/DataTypeExtractorScalaTest.scala | 52 +-
.../TypeInferenceExtractorScalaTest.scala | 66 +-
flink-table/flink-table-planner/pom.xml | 25 +-
.../table/expressions/ExpressionParserImpl.scala | 341 +--
.../table/planner/calcite/CalciteConfig.scala | 92 +-
.../flink/table/planner/calcite/FlinkContext.scala | 9 +-
.../table/planner/calcite/FlinkContextImpl.scala | 1 -
.../table/planner/calcite/FlinkLocalRef.scala | 29 +-
.../planner/calcite/FlinkLogicalRelFactories.scala | 88 +-
.../table/planner/calcite/FlinkPlannerImpl.scala | 117 +-
.../table/planner/calcite/FlinkRelFactories.scala | 35 +-
.../calcite/FlinkRelOptClusterFactory.scala | 9 +-
.../table/planner/calcite/FlinkTypeFactory.scala | 238 +-
.../table/planner/calcite/FlinkTypeSystem.scala | 56 +-
.../planner/calcite/PreValidateReWriter.scala | 214 +-
.../table/planner/codegen/CalcCodeGenerator.scala | 22 +-
.../table/planner/codegen/CodeGenException.scala | 5 +-
.../flink/table/planner/codegen/CodeGenUtils.scala | 386 ++--
.../planner/codegen/CodeGeneratorContext.scala | 526 +++--
.../planner/codegen/CollectorCodeGenerator.scala | 70 +-
.../planner/codegen/CorrelateCodeGenerator.scala | 29 +-
.../planner/codegen/EqualiserCodeGenerator.scala | 28 +-
.../planner/codegen/ExpandCodeGenerator.scala | 28 +-
.../table/planner/codegen/ExprCodeGenerator.scala | 390 ++--
.../table/planner/codegen/ExpressionReducer.scala | 323 ++-
.../codegen/ExternalGeneratedExpression.scala | 51 +-
.../planner/codegen/FunctionCodeGenerator.scala | 221 +-
.../table/planner/codegen/GenerateUtils.scala | 401 ++--
.../planner/codegen/GeneratedExpression.scala | 69 +-
.../table/planner/codegen/HashCodeGenerator.scala | 76 +-
.../flink/table/planner/codegen/Indenter.scala | 10 +-
.../planner/codegen/InputFormatCodeGenerator.scala | 66 +-
.../table/planner/codegen/JsonGenerateUtils.scala | 75 +-
.../planner/codegen/LongHashJoinGenerator.scala | 99 +-
.../planner/codegen/LookupJoinCodeGenerator.scala | 215 +-
.../table/planner/codegen/MatchCodeGenerator.scala | 376 ++--
.../codegen/NestedLoopJoinCodeGenerator.scala | 119 +-
.../planner/codegen/OperatorCodeGenerator.scala | 91 +-
.../planner/codegen/ProjectionCodeGenerator.scala | 58 +-
.../table/planner/codegen/SinkCodeGenerator.scala | 38 +-
.../planner/codegen/ValuesCodeGenerator.scala | 18 +-
.../codegen/WatermarkGeneratorCodeGenerator.scala | 41 +-
.../table/planner/codegen/agg/AggCodeGen.scala | 16 +-
.../codegen/agg/AggsHandlerCodeGenerator.scala | 492 ++--
.../codegen/agg/DeclarativeAggCodeGen.scala | 131 +-
.../planner/codegen/agg/DistinctAggCodeGen.scala | 256 ++-
.../planner/codegen/agg/ImperativeAggCodeGen.scala | 145 +-
.../codegen/agg/batch/AggCodeGenHelper.scala | 617 +++--
.../agg/batch/AggWithoutKeysCodeGenerator.scala | 10 +-
.../codegen/agg/batch/HashAggCodeGenHelper.scala | 275 +--
.../codegen/agg/batch/HashAggCodeGenerator.scala | 67 +-
.../agg/batch/HashWindowCodeGenerator.scala | 246 +-
.../codegen/agg/batch/SortAggCodeGenerator.scala | 33 +-
.../agg/batch/SortWindowCodeGenerator.scala | 111 +-
.../codegen/agg/batch/WindowCodeGenerator.scala | 164 +-
.../codegen/calls/BridgingFunctionGenUtil.scala | 203 +-
.../codegen/calls/BridgingSqlFunctionCallGen.scala | 9 +-
.../planner/codegen/calls/BuiltInMethods.scala | 459 ++--
.../planner/codegen/calls/CallGenerator.scala | 13 +-
.../planner/codegen/calls/ConstantCallGen.scala | 7 +-
.../codegen/calls/CurrentTimePointCallGen.scala | 11 +-
.../table/planner/codegen/calls/DivCallGen.scala | 5 +-
.../planner/codegen/calls/ExtractCallGen.scala | 110 +-
.../planner/codegen/calls/FloorCeilCallGen.scala | 13 +-
.../planner/codegen/calls/FunctionGenerator.scala | 813 ++-----
.../planner/codegen/calls/HashCodeCallGen.scala | 5 +-
.../table/planner/codegen/calls/IfCallGen.scala | 54 +-
.../planner/codegen/calls/JsonArrayCallGen.scala | 51 +-
.../planner/codegen/calls/JsonObjectCallGen.scala | 75 +-
.../planner/codegen/calls/JsonStringCallGen.scala | 5 +-
.../planner/codegen/calls/JsonValueCallGen.scala | 89 +-
.../table/planner/codegen/calls/LikeCallGen.scala | 29 +-
.../planner/codegen/calls/MethodCallGen.scala | 59 +-
.../table/planner/codegen/calls/NotCallGen.scala | 11 +-
.../table/planner/codegen/calls/PrintCallGen.scala | 7 +-
.../table/planner/codegen/calls/RandCallGen.scala | 16 +-
.../codegen/calls/ScalarFunctionCallGen.scala | 85 +-
.../planner/codegen/calls/ScalarOperatorGens.scala | 856 ++++---
.../planner/codegen/calls/SearchOperatorGen.scala | 67 +-
.../planner/codegen/calls/StringCallGen.scala | 350 ++-
.../codegen/calls/TableFunctionCallGen.scala | 56 +-
.../codegen/calls/TimestampDiffCallGen.scala | 20 +-
...ltiFieldRangeBoundComparatorCodeGenerator.scala | 15 +-
.../over/RangeBoundComparatorCodeGenerator.scala | 56 +-
.../codegen/sort/ComparatorCodeGenerator.scala | 37 +-
.../planner/codegen/sort/SortCodeGenerator.scala | 99 +-
.../table/planner/delegation/BatchPlanner.scala | 34 +-
.../table/planner/delegation/PlannerBase.scala | 145 +-
.../table/planner/delegation/StreamPlanner.scala | 68 +-
.../planner/expressions/ExpressionBridge.scala | 5 +-
.../table/planner/expressions/InputTypeSpec.scala | 38 +-
.../planner/expressions/PlannerExpression.scala | 24 +-
.../expressions/PlannerExpressionConverter.scala | 68 +-
.../expressions/PlannerExpressionUtils.scala | 5 +-
.../table/planner/expressions/Reinterpret.scala | 16 +-
.../flink/table/planner/expressions/TreeNode.scala | 75 +-
.../table/planner/expressions/aggregations.scala | 53 +-
.../flink/table/planner/expressions/call.scala | 99 +-
.../table/planner/expressions/collection.scala | 13 +-
.../planner/expressions/fieldExpression.scala | 25 +-
.../flink/table/planner/expressions/literals.scala | 23 +-
.../table/planner/expressions/overOffsets.scala | 1 -
.../flink/table/planner/expressions/subquery.scala | 9 +-
.../flink/table/planner/expressions/symbols.scala | 27 +-
.../flink/table/planner/expressions/time.scala | 175 +-
.../planner/expressions/windowProperties.scala | 1 -
.../planner/functions/utils/AggSqlFunction.scala | 105 +-
.../functions/utils/ScalarSqlFunction.scala | 77 +-
.../planner/functions/utils/TableSqlFunction.scala | 144 +-
.../functions/utils/UserDefinedFunctionUtils.scala | 729 +++---
.../flink/table/planner/plan/cost/FlinkCost.scala | 272 ++-
.../table/planner/plan/cost/FlinkCostBase.scala | 13 +-
.../table/planner/plan/cost/FlinkCostFactory.scala | 5 +-
.../planner/plan/cost/FlinkCostFactoryBase.scala | 15 +-
.../planner/plan/logical/MatchRecognize.scala | 23 +-
.../table/planner/plan/logical/groupWindows.scala | 26 +-
.../metadata/AggCallSelectivityEstimator.scala | 177 +-
.../metadata/FlinkDefaultRelMetadataProvider.scala | 1 -
.../plan/metadata/FlinkRelMdColumnInterval.scala | 470 ++--
.../plan/metadata/FlinkRelMdColumnNullCount.scala | 170 +-
.../metadata/FlinkRelMdColumnOriginNullCount.scala | 20 +-
.../plan/metadata/FlinkRelMdColumnUniqueness.scala | 232 +-
.../plan/metadata/FlinkRelMdCumulativeCost.scala | 16 +-
.../plan/metadata/FlinkRelMdDistinctRowCount.scala | 156 +-
.../plan/metadata/FlinkRelMdDistribution.scala | 32 +-
.../FlinkRelMdFilteredColumnInterval.scala | 173 +-
.../metadata/FlinkRelMdModifiedMonotonicity.scala | 130 +-
.../metadata/FlinkRelMdNonCumulativeCost.scala | 10 +-
.../FlinkRelMdPercentageOriginalRows.scala | 29 +-
.../plan/metadata/FlinkRelMdPopulationSize.scala | 152 +-
.../planner/plan/metadata/FlinkRelMdRowCount.scala | 82 +-
.../plan/metadata/FlinkRelMdSelectivity.scala | 74 +-
.../planner/plan/metadata/FlinkRelMdSize.scala | 172 +-
.../plan/metadata/FlinkRelMdUniqueGroups.scala | 115 +-
.../plan/metadata/FlinkRelMdUniqueKeys.scala | 173 +-
.../plan/metadata/FlinkRelMdUpsertKeys.scala | 102 +-
.../plan/metadata/FlinkRelMdWindowProperties.scala | 122 +-
.../plan/metadata/SelectivityEstimator.scala | 582 ++---
.../planner/plan/nodes/FlinkConventions.scala | 13 +-
.../table/planner/plan/nodes/FlinkRelNode.scala | 36 +-
.../table/planner/plan/nodes/calcite/Expand.scala | 36 +-
.../planner/plan/nodes/calcite/LegacySink.scala | 27 +-
.../planner/plan/nodes/calcite/LogicalExpand.scala | 9 +-
.../plan/nodes/calcite/LogicalLegacySink.scala | 32 +-
.../planner/plan/nodes/calcite/LogicalRank.scala | 8 +-
.../planner/plan/nodes/calcite/LogicalSink.scala | 12 +-
.../plan/nodes/calcite/LogicalTableAggregate.scala | 11 +-
.../nodes/calcite/LogicalWatermarkAssigner.scala | 10 +-
.../nodes/calcite/LogicalWindowAggregate.scala | 3 +-
.../calcite/LogicalWindowTableAggregate.scala | 16 +-
.../table/planner/plan/nodes/calcite/Rank.scala | 71 +-
.../table/planner/plan/nodes/calcite/Sink.scala | 28 +-
.../plan/nodes/calcite/TableAggregate.scala | 25 +-
.../plan/nodes/calcite/WatermarkAssigner.scala | 35 +-
.../plan/nodes/calcite/WindowAggregate.scala | 45 +-
.../plan/nodes/calcite/WindowTableAggregate.scala | 23 +-
.../planner/plan/nodes/common/CommonCalc.scala | 44 +-
.../nodes/common/CommonIntermediateTableScan.scala | 9 +-
.../common/CommonPhysicalWindowTableFunction.scala | 10 +-
.../plan/nodes/logical/FlinkLogicalAggregate.scala | 37 +-
.../plan/nodes/logical/FlinkLogicalCalc.scala | 21 +-
.../plan/nodes/logical/FlinkLogicalCorrelate.scala | 15 +-
.../logical/FlinkLogicalDataStreamTableScan.scala | 34 +-
.../nodes/logical/FlinkLogicalDistribution.scala | 52 +-
.../plan/nodes/logical/FlinkLogicalExpand.scala | 19 +-
.../FlinkLogicalIntermediateTableScan.scala | 28 +-
.../plan/nodes/logical/FlinkLogicalIntersect.scala | 11 +-
.../plan/nodes/logical/FlinkLogicalJoin.scala | 22 +-
.../nodes/logical/FlinkLogicalLegacySink.scala | 25 +-
.../FlinkLogicalLegacyTableSourceScan.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalMatch.scala | 12 +-
.../plan/nodes/logical/FlinkLogicalMinus.scala | 7 +-
.../nodes/logical/FlinkLogicalOverAggregate.scala | 43 +-
.../plan/nodes/logical/FlinkLogicalRank.scala | 31 +-
.../plan/nodes/logical/FlinkLogicalRel.scala | 9 +-
.../plan/nodes/logical/FlinkLogicalSink.scala | 7 +-
.../plan/nodes/logical/FlinkLogicalSnapshot.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalSort.scala | 47 +-
.../nodes/logical/FlinkLogicalTableAggregate.scala | 16 +-
.../logical/FlinkLogicalTableFunctionScan.scala | 9 +-
.../logical/FlinkLogicalTableSourceScan.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalUnion.scala | 13 +-
.../plan/nodes/logical/FlinkLogicalValues.scala | 20 +-
.../logical/FlinkLogicalWatermarkAssigner.scala | 28 +-
.../logical/FlinkLogicalWindowAggregate.scala | 3 +-
.../logical/FlinkLogicalWindowTableAggregate.scala | 7 +-
.../plan/nodes/physical/FlinkPhysicalRel.scala | 24 +-
.../batch/BatchPhysicalBoundedStreamScan.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalCalc.scala | 7 +-
.../physical/batch/BatchPhysicalCalcBase.scala | 13 +-
.../physical/batch/BatchPhysicalCorrelate.scala | 22 +-
.../batch/BatchPhysicalCorrelateBase.scala | 42 +-
.../physical/batch/BatchPhysicalExchange.scala | 16 +-
.../nodes/physical/batch/BatchPhysicalExpand.scala | 7 +-
.../batch/BatchPhysicalGroupAggregateBase.scala | 21 +-
.../batch/BatchPhysicalHashAggregate.scala | 44 +-
.../batch/BatchPhysicalHashAggregateBase.scala | 11 +-
.../physical/batch/BatchPhysicalHashJoin.scala | 21 +-
.../batch/BatchPhysicalHashWindowAggregate.scala | 12 +-
.../BatchPhysicalHashWindowAggregateBase.scala | 7 +-
.../batch/BatchPhysicalIntermediateTableScan.scala | 7 +-
.../physical/batch/BatchPhysicalJoinBase.scala | 134 +-
.../physical/batch/BatchPhysicalLegacySink.scala | 6 +-
.../batch/BatchPhysicalLegacyTableSourceScan.scala | 7 +-
.../nodes/physical/batch/BatchPhysicalLimit.scala | 11 +-
.../batch/BatchPhysicalLocalHashAggregate.scala | 35 +-
.../BatchPhysicalLocalHashWindowAggregate.scala | 9 +-
.../batch/BatchPhysicalLocalSortAggregate.scala | 38 +-
.../BatchPhysicalLocalSortWindowAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalLookupJoin.scala | 7 +-
.../batch/BatchPhysicalNestedLoopJoin.scala | 21 +-
.../batch/BatchPhysicalOverAggregate.scala | 7 +-
.../batch/BatchPhysicalOverAggregateBase.scala | 72 +-
.../physical/batch/BatchPhysicalPythonCalc.scala | 16 +-
.../batch/BatchPhysicalPythonCorrelate.scala | 18 +-
.../batch/BatchPhysicalPythonGroupAggregate.scala | 44 +-
.../BatchPhysicalPythonGroupWindowAggregate.scala | 9 +-
.../batch/BatchPhysicalPythonOverAggregate.scala | 3 +-
.../nodes/physical/batch/BatchPhysicalRank.scala | 16 +-
.../nodes/physical/batch/BatchPhysicalRel.scala | 10 +-
.../nodes/physical/batch/BatchPhysicalSink.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalSort.scala | 13 +-
.../batch/BatchPhysicalSortAggregate.scala | 44 +-
.../batch/BatchPhysicalSortAggregateBase.scala | 5 +-
.../physical/batch/BatchPhysicalSortLimit.scala | 21 +-
.../batch/BatchPhysicalSortMergeJoin.scala | 58 +-
.../batch/BatchPhysicalSortWindowAggregate.scala | 5 +-
.../BatchPhysicalSortWindowAggregateBase.scala | 3 +-
.../batch/BatchPhysicalTableSourceScan.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalUnion.scala | 21 +-
.../nodes/physical/batch/BatchPhysicalValues.scala | 10 +-
.../batch/BatchPhysicalWindowAggregateBase.scala | 41 +-
.../batch/BatchPhysicalWindowTableFunction.scala | 23 +-
.../physical/common/CommonPhysicalExchange.scala | 20 +-
.../nodes/physical/common/CommonPhysicalJoin.scala | 43 +-
.../CommonPhysicalLegacyTableSourceScan.scala | 10 +-
.../physical/common/CommonPhysicalLookupJoin.scala | 193 +-
.../common/CommonPhysicalTableSourceScan.scala | 11 +-
.../nodes/physical/stream/StreamPhysicalCalc.scala | 9 +-
.../physical/stream/StreamPhysicalCalcBase.scala | 7 +-
.../stream/StreamPhysicalChangelogNormalize.scala | 10 +-
.../physical/stream/StreamPhysicalCorrelate.scala | 22 +-
.../stream/StreamPhysicalCorrelateBase.scala | 30 +-
.../stream/StreamPhysicalDataStreamScan.scala | 13 +-
.../stream/StreamPhysicalDeduplicate.scala | 12 +-
.../stream/StreamPhysicalDropUpdateBefore.scala | 24 +-
.../physical/stream/StreamPhysicalExchange.scala | 7 +-
.../physical/stream/StreamPhysicalExpand.scala | 9 +-
.../StreamPhysicalGlobalGroupAggregate.scala | 36 +-
.../StreamPhysicalGlobalWindowAggregate.scala | 52 +-
.../stream/StreamPhysicalGroupAggregate.scala | 22 +-
.../stream/StreamPhysicalGroupAggregateBase.scala | 45 +-
.../stream/StreamPhysicalGroupTableAggregate.scala | 8 +-
.../StreamPhysicalGroupTableAggregateBase.scala | 27 +-
.../StreamPhysicalGroupWindowAggregate.scala | 9 +-
.../StreamPhysicalGroupWindowAggregateBase.scala | 39 +-
.../StreamPhysicalGroupWindowTableAggregate.scala | 5 +-
.../StreamPhysicalIncrementalGroupAggregate.scala | 36 +-
.../StreamPhysicalIntermediateTableScan.scala | 7 +-
.../stream/StreamPhysicalIntervalJoin.scala | 63 +-
.../nodes/physical/stream/StreamPhysicalJoin.scala | 59 +-
.../physical/stream/StreamPhysicalLegacySink.scala | 6 +-
.../StreamPhysicalLegacyTableSourceScan.scala | 5 +-
.../physical/stream/StreamPhysicalLimit.scala | 18 +-
.../stream/StreamPhysicalLocalGroupAggregate.scala | 27 +-
.../StreamPhysicalLocalWindowAggregate.scala | 50 +-
.../physical/stream/StreamPhysicalLookupJoin.scala | 6 +-
.../physical/stream/StreamPhysicalMatch.scala | 45 +-
.../stream/StreamPhysicalMiniBatchAssigner.scala | 11 +-
.../stream/StreamPhysicalOverAggregate.scala | 15 +-
.../stream/StreamPhysicalOverAggregateBase.scala | 32 +-
.../physical/stream/StreamPhysicalPythonCalc.scala | 16 +-
.../stream/StreamPhysicalPythonCorrelate.scala | 18 +-
.../StreamPhysicalPythonGroupAggregate.scala | 29 +-
.../StreamPhysicalPythonGroupTableAggregate.scala | 8 +-
.../StreamPhysicalPythonGroupWindowAggregate.scala | 9 +-
.../stream/StreamPhysicalPythonOverAggregate.scala | 15 +-
.../nodes/physical/stream/StreamPhysicalRank.scala | 7 +-
.../nodes/physical/stream/StreamPhysicalRel.scala | 9 +-
.../nodes/physical/stream/StreamPhysicalSink.scala | 19 +-
.../nodes/physical/stream/StreamPhysicalSort.scala | 25 +-
.../physical/stream/StreamPhysicalSortLimit.scala | 20 +-
.../stream/StreamPhysicalTableSourceScan.scala | 1 -
.../stream/StreamPhysicalTemporalJoin.scala | 67 +-
.../stream/StreamPhysicalTemporalSort.scala | 15 +-
.../physical/stream/StreamPhysicalUnion.scala | 9 +-
.../physical/stream/StreamPhysicalValues.scala | 7 +-
.../stream/StreamPhysicalWatermarkAssigner.scala | 25 +-
.../stream/StreamPhysicalWindowAggregate.scala | 32 +-
.../stream/StreamPhysicalWindowDeduplicate.scala | 11 +-
.../physical/stream/StreamPhysicalWindowJoin.scala | 36 +-
.../physical/stream/StreamPhysicalWindowRank.scala | 6 +-
.../stream/StreamPhysicalWindowTableFunction.scala | 16 +-
.../BatchCommonSubGraphBasedOptimizer.scala | 59 +-
.../optimize/CommonSubGraphBasedOptimizer.scala | 105 +-
.../table/planner/plan/optimize/Optimizer.scala | 21 +-
.../table/planner/plan/optimize/RelNodeBlock.scala | 249 +-
.../StreamCommonSubGraphBasedOptimizer.scala | 144 +-
.../optimize/program/BatchOptimizeContext.scala | 9 +-
.../plan/optimize/program/FlinkBatchProgram.scala | 209 +-
.../optimize/program/FlinkChainedProgram.scala | 87 +-
.../FlinkChangelogModeInferenceProgram.scala | 716 +++---
.../optimize/program/FlinkDecorrelateProgram.scala | 29 +-
.../plan/optimize/program/FlinkGroupProgram.scala | 30 +-
.../plan/optimize/program/FlinkHepProgram.scala | 37 +-
.../optimize/program/FlinkHepRuleSetProgram.scala | 109 +-
.../FlinkMiniBatchIntervalTraitInitProgram.scala | 9 +-
.../optimize/program/FlinkOptimizeContext.scala | 15 +-
.../optimize/program/FlinkOptimizeProgram.scala | 16 +-
.../program/FlinkRelTimeIndicatorProgram.scala | 10 +-
.../optimize/program/FlinkRuleSetProgram.scala | 36 +-
.../plan/optimize/program/FlinkStreamProgram.scala | 219 +-
.../optimize/program/FlinkVolcanoProgram.scala | 39 +-
.../optimize/program/StreamOptimizeContext.scala | 13 +-
.../table/planner/plan/reuse/SubplanReuser.scala | 146 +-
.../planner/plan/rules/FlinkBatchRuleSets.scala | 150 +-
.../planner/plan/rules/FlinkStreamRuleSets.scala | 206 +-
.../rules/common/CommonTemporalTableJoinRule.scala | 29 +-
.../logical/AggregateReduceGroupingRule.scala | 48 +-
.../logical/BatchLogicalWindowAggregateRule.scala | 20 +-
.../plan/rules/logical/CalcRankTransposeRule.scala | 44 +-
.../rules/logical/CalcSnapshotTransposeRule.scala | 14 +-
.../ConstantRankNumberColumnRemoveRule.scala | 9 +-
.../rules/logical/ConvertToNotInOrInRule.scala | 91 +-
.../rules/logical/CorrelateSortToRankRule.scala | 58 +-
.../rules/logical/DecomposeGroupingSetsRule.scala | 333 ++-
.../logical/EnumerableToLogicalTableScan.scala | 19 +-
.../plan/rules/logical/FlinkCalcMergeRule.scala | 63 +-
.../plan/rules/logical/FlinkLimit0RemoveRule.scala | 12 +-
.../plan/rules/logical/FlinkLogicalRankRule.scala | 77 +-
.../plan/rules/logical/FlinkPruneEmptyRules.scala | 33 +-
.../rules/logical/FlinkRewriteSubQueryRule.scala | 69 +-
.../rules/logical/FlinkSubQueryRemoveRule.scala | 85 +-
.../JoinConditionEqualityTransferRule.scala | 91 +-
.../logical/JoinConditionTypeCoerceRule.scala | 52 +-
.../JoinDependentConditionDerivationRule.scala | 127 +-
.../rules/logical/JoinDeriveNullFilterRule.scala | 49 +-
...relateToJoinFromTemporalTableFunctionRule.scala | 102 +-
...gicalCorrelateToJoinFromTemporalTableRule.scala | 150 +-
.../plan/rules/logical/LogicalUnnestRule.scala | 36 +-
.../logical/LogicalWindowAggregateRuleBase.scala | 196 +-
.../logical/ProjectSemiAntiJoinTransposeRule.scala | 46 +-
.../rules/logical/PruneAggregateCallRule.scala | 78 +-
.../PushFilterIntoLegacyTableSourceScanRule.scala | 42 +-
.../PushLimitIntoLegacyTableSourceScanRule.scala | 61 +-
...ushPartitionIntoLegacyTableSourceScanRule.scala | 85 +-
.../PushProjectIntoLegacyTableSourceScanRule.scala | 69 +-
.../plan/rules/logical/PythonCalcSplitRule.scala | 288 +--
.../logical/ReplaceIntersectWithSemiJoinRule.scala | 22 +-
.../logical/ReplaceMinusWithAntiJoinRule.scala | 22 +-
.../rules/logical/RewriteIntersectAllRule.scala | 147 +-
.../plan/rules/logical/RewriteMinusAllRule.scala | 107 +-
.../logical/RewriteMultiJoinConditionRule.scala | 53 +-
.../logical/SimplifyFilterConditionRule.scala | 36 +-
.../rules/logical/SimplifyJoinConditionRule.scala | 17 +-
.../plan/rules/logical/SplitAggregateRule.scala | 465 ++--
.../SplitPythonConditionFromCorrelateRule.scala | 45 +-
.../logical/SplitPythonConditionFromJoinRule.scala | 27 +-
.../logical/StreamLogicalWindowAggregateRule.scala | 27 +-
.../TemporalJoinRewriteWithUniqueKeyRule.scala | 79 +-
.../WindowAggregateReduceFunctionsRule.scala | 19 +-
.../rules/logical/WindowGroupReorderRule.scala | 57 +-
.../plan/rules/logical/WindowPropertiesRule.scala | 103 +-
.../rules/physical/FlinkExpandConversionRule.scala | 21 +-
.../physical/batch/BatchPhysicalAggRuleBase.scala | 51 +-
.../batch/BatchPhysicalBoundedStreamScanRule.scala | 9 +-
.../physical/batch/BatchPhysicalCalcRule.scala | 15 +-
...atchPhysicalConstantTableFunctionScanRule.scala | 32 +-
.../batch/BatchPhysicalCorrelateRule.scala | 14 +-
.../batch/BatchPhysicalDistributionRule.scala | 31 +-
.../physical/batch/BatchPhysicalExpandRule.scala | 5 +-
.../physical/batch/BatchPhysicalHashAggRule.scala | 120 +-
.../physical/batch/BatchPhysicalHashJoinRule.scala | 64 +-
.../BatchPhysicalIntermediateTableScanRule.scala | 1 -
.../physical/batch/BatchPhysicalJoinRuleBase.scala | 38 +-
.../batch/BatchPhysicalLegacySinkRule.scala | 25 +-
.../BatchPhysicalLegacyTableSourceScanRule.scala | 1 -
.../physical/batch/BatchPhysicalLimitRule.scala | 34 +-
.../batch/BatchPhysicalLookupJoinRule.scala | 16 +-
.../batch/BatchPhysicalNestedLoopJoinRule.scala | 11 +-
.../batch/BatchPhysicalOverAggregateRule.scala | 63 +-
.../batch/BatchPhysicalPythonCalcRule.scala | 19 +-
.../physical/batch/BatchPhysicalRankRule.scala | 24 +-
.../batch/BatchPhysicalSingleRowJoinRule.scala | 22 +-
.../physical/batch/BatchPhysicalSinkRule.scala | 35 +-
.../physical/batch/BatchPhysicalSortAggRule.scala | 104 +-
.../batch/BatchPhysicalSortLimitRule.scala | 36 +-
.../batch/BatchPhysicalSortMergeJoinRule.scala | 73 +-
.../physical/batch/BatchPhysicalSortRule.scala | 31 +-
.../batch/BatchPhysicalTableSourceScanRule.scala | 5 +-
.../physical/batch/BatchPhysicalUnionRule.scala | 12 +-
.../physical/batch/BatchPhysicalValuesRule.scala | 22 +-
.../batch/BatchPhysicalWindowAggregateRule.scala | 108 +-
.../BatchPhysicalWindowTableFunctionRule.scala | 16 +-
.../physical/batch/EnforceLocalAggRuleBase.scala | 27 +-
.../physical/batch/EnforceLocalHashAggRule.scala | 15 +-
.../physical/batch/EnforceLocalSortAggRule.scala | 73 +-
.../batch/RemoveRedundantLocalHashAggRule.scala | 16 +-
.../batch/RemoveRedundantLocalRankRule.scala | 30 +-
.../batch/RemoveRedundantLocalSortAggRule.scala | 46 +-
.../physical/common/CommonLookupJoinRule.scala | 40 +-
.../ExpandWindowTableFunctionTransposeRule.scala | 101 +-
.../physical/stream/IncrementalAggregateRule.scala | 55 +-
.../stream/MiniBatchIntervalInferRule.scala | 115 +-
...indowTableFunctionIntoWindowAggregateRule.scala | 46 +-
.../physical/stream/StreamPhysicalCalcRule.scala | 12 +-
...reamPhysicalConstantTableFunctionScanRule.scala | 32 +-
.../stream/StreamPhysicalCorrelateRule.scala | 13 +-
.../stream/StreamPhysicalDataStreamScanRule.scala | 5 +-
.../stream/StreamPhysicalDeduplicateRule.scala | 54 +-
.../physical/stream/StreamPhysicalExpandRule.scala | 5 +-
.../stream/StreamPhysicalGroupAggregateRule.scala | 8 +-
.../StreamPhysicalGroupTableAggregateRule.scala | 7 +-
.../StreamPhysicalGroupWindowAggregateRule.scala | 5 +-
...reamPhysicalGroupWindowTableAggregateRule.scala | 7 +-
.../StreamPhysicalIntermediateTableScanRule.scala | 1 -
.../stream/StreamPhysicalIntervalJoinRule.scala | 40 +-
.../physical/stream/StreamPhysicalJoinRule.scala | 14 +-
.../stream/StreamPhysicalJoinRuleBase.scala | 17 +-
.../stream/StreamPhysicalLegacySinkRule.scala | 23 +-
.../StreamPhysicalLegacyTableSourceScanRule.scala | 1 -
.../physical/stream/StreamPhysicalLimitRule.scala | 9 +-
.../stream/StreamPhysicalLookupJoinRule.scala | 24 +-
.../physical/stream/StreamPhysicalMatchRule.scala | 28 +-
.../stream/StreamPhysicalOverAggregateRule.scala | 7 +-
.../stream/StreamPhysicalPythonCalcRule.scala | 12 +-
.../physical/stream/StreamPhysicalRankRule.scala | 7 +-
.../physical/stream/StreamPhysicalSinkRule.scala | 36 +-
.../stream/StreamPhysicalSortLimitRule.scala | 9 +-
.../physical/stream/StreamPhysicalSortRule.scala | 14 +-
.../stream/StreamPhysicalTableSourceScanRule.scala | 22 +-
.../stream/StreamPhysicalTemporalJoinRule.scala | 5 +-
.../stream/StreamPhysicalTemporalSortRule.scala | 34 +-
.../physical/stream/StreamPhysicalUnionRule.scala | 12 +-
.../physical/stream/StreamPhysicalValuesRule.scala | 11 +-
.../StreamPhysicalWatermarkAssignerRule.scala | 15 +-
.../stream/StreamPhysicalWindowAggregateRule.scala | 69 +-
.../StreamPhysicalWindowDeduplicateRule.scala | 14 +-
.../stream/StreamPhysicalWindowJoinRule.scala | 18 +-
.../stream/StreamPhysicalWindowRankRule.scala | 12 +-
.../StreamPhysicalWindowTableFunctionRule.scala | 16 +-
.../stream/TwoStageOptimizedAggregateRule.scala | 71 +-
.../planner/plan/schema/DataStreamTable.scala | 37 +-
.../schema/DeferredTypeFlinkTableFunction.scala | 20 +-
.../planner/plan/schema/FlinkTableFunction.scala | 21 +-
.../planner/plan/schema/GenericRelDataType.scala | 35 +-
.../planner/plan/schema/IntermediateRelTable.scala | 25 +-
.../plan/schema/LegacyCatalogSourceTable.scala | 126 +-
.../plan/schema/LegacyTableSourceTable.scala | 90 +-
.../planner/plan/schema/TableSourceTable.scala | 66 +-
.../plan/schema/TimeIndicatorRelDataType.scala | 15 +-
.../plan/schema/TypedFlinkTableFunction.scala | 26 +-
.../table/planner/plan/stats/FlinkStatistic.scala | 102 +-
.../table/planner/plan/stats/ValueInterval.scala | 158 +-
.../planner/plan/trait/FlinkRelDistribution.scala | 96 +-
.../plan/trait/MiniBatchIntervalTrait.scala | 13 +-
.../plan/trait/MiniBatchIntervalTraitDef.scala | 1 -
.../planner/plan/trait/ModifyKindSetTrait.scala | 25 +-
.../planner/plan/trait/ModifyKindSetTraitDef.scala | 3 -
.../plan/trait/RelModifiedMonotonicity.scala | 8 +-
.../flink/table/planner/plan/trait/TraitUtil.scala | 42 +-
.../table/planner/plan/trait/UpdateKindTrait.scala | 34 +-
.../planner/plan/trait/UpdateKindTraitDef.scala | 5 +-
.../planner/plan/utils/AggFunctionFactory.scala | 120 +-
.../table/planner/plan/utils/AggregateUtil.scala | 783 +++----
.../planner/plan/utils/ChangelogPlanUtils.scala | 37 +-
.../planner/plan/utils/ColumnIntervalUtil.scala | 123 +-
.../table/planner/plan/utils/CorrelateUtil.scala | 69 +-
.../table/planner/plan/utils/ExpandUtil.scala | 170 +-
.../table/planner/plan/utils/FlinkRelMdUtil.scala | 421 ++--
.../table/planner/plan/utils/FlinkRelOptUtil.scala | 371 +--
.../table/planner/plan/utils/FlinkRexUtil.scala | 255 +--
.../planner/plan/utils/IntervalJoinUtil.scala | 223 +-
.../flink/table/planner/plan/utils/JoinUtil.scala | 109 +-
.../flink/table/planner/plan/utils/MatchUtil.scala | 33 +-
.../planner/plan/utils/NestedProjectionUtil.scala | 177 +-
.../planner/plan/utils/OverAggregateUtil.scala | 119 +-
.../table/planner/plan/utils/PartitionPruner.scala | 78 +-
.../table/planner/plan/utils/PythonUtil.scala | 116 +-
.../flink/table/planner/plan/utils/RankUtil.scala | 146 +-
.../table/planner/plan/utils/RelExplainUtil.scala | 813 +++----
.../table/planner/plan/utils/RelShuttles.scala | 71 +-
.../planner/plan/utils/RelTreeWriterImpl.scala | 29 +-
.../planner/plan/utils/RexDefaultVisitor.scala | 7 +-
.../planner/plan/utils/RexNodeExtractor.scala | 254 ++-
.../table/planner/plan/utils/RexNodeRewriter.scala | 32 +-
.../flink/table/planner/plan/utils/ScanUtil.scala | 65 +-
.../planner/plan/utils/SetOpRewriteUtil.scala | 46 +-
.../flink/table/planner/plan/utils/SortUtil.scala | 67 +-
.../planner/plan/utils/TemporalJoinUtil.scala | 150 +-
.../planner/plan/utils/UpdatingPlanChecker.scala | 11 +-
.../planner/plan/utils/WindowEmitStrategy.scala | 65 +-
.../table/planner/plan/utils/WindowJoinUtil.scala | 185 +-
.../table/planner/plan/utils/WindowUtil.scala | 161 +-
.../table/planner/plan/utils/aggregation.scala | 102 +-
.../table/planner/sinks/CollectTableSink.scala | 14 +-
.../table/planner/sinks/DataStreamTableSink.scala | 35 +-
.../flink/table/planner/sinks/TableSinkUtils.scala | 210 +-
.../table/planner/sources/TableSourceUtil.scala | 199 +-
.../planner/typeutils/LegacyDataViewUtils.scala | 54 +-
.../table/planner/typeutils/TypeCoercion.scala | 199 +-
.../planner/typeutils/TypeInfoCheckUtils.scala | 74 +-
.../planner/utils/JavaScalaConversionUtil.scala | 7 +-
.../apache/flink/table/planner/utils/Logging.scala | 5 +-
.../table/planner/validate/ValidationResult.scala | 19 +-
.../apache/flink/table/api/ExecuteSqlTest.scala | 46 +-
.../flink/table/api/TableEnvironmentITCase.scala | 120 +-
.../flink/table/api/TableEnvironmentTest.scala | 423 ++--
.../org/apache/flink/table/api/TableITCase.scala | 73 +-
.../apache/flink/table/api/batch/ExplainTest.scala | 11 +-
.../flink/table/api/stream/ExplainTest.scala | 80 +-
.../validation/MatchRecognizeValidationTest.scala | 40 +-
.../sql/validation/OverWindowValidationTest.scala | 9 +-
.../planner/calcite/CalciteConfigBuilderTest.scala | 22 +-
.../planner/calcite/FlinkTypeFactoryTest.scala | 31 +-
.../table/planner/catalog/CatalogTableITCase.scala | 251 ++-
.../table/planner/catalog/CatalogTableTest.scala | 11 +-
.../table/planner/catalog/CatalogViewITCase.scala | 197 +-
.../planner/codegen/HashCodeGeneratorTest.scala | 35 +-
.../codegen/ProjectionCodeGeneratorTest.scala | 102 +-
.../codegen/WatermarkGeneratorCodeGenTest.scala | 51 +-
.../table/planner/codegen/agg/AggTestBase.scala | 20 +-
.../codegen/agg/AggsHandlerCodeGeneratorTest.scala | 37 +-
.../codegen/agg/batch/AggWithoutKeysTest.scala | 73 +-
.../codegen/agg/batch/BatchAggTestBase.scala | 46 +-
.../agg/batch/HashAggCodeGeneratorTest.scala | 89 +-
.../agg/batch/SortAggCodeGeneratorTest.scala | 96 +-
.../table/planner/expressions/ArrayTypeTest.scala | 248 +-
.../planner/expressions/CompositeAccessTest.scala | 36 +-
.../planner/expressions/DecimalTypeTest.scala | 641 ++----
.../planner/expressions/KeywordParseTest.scala | 6 +-
.../table/planner/expressions/LiteralTest.scala | 87 +-
.../table/planner/expressions/MapTypeTest.scala | 207 +-
.../expressions/NonDeterministicTests.scala | 70 +-
.../table/planner/expressions/RowTypeTest.scala | 21 +-
.../planner/expressions/ScalarFunctionsTest.scala | 2381 ++++++--------------
.../planner/expressions/ScalarOperatorsTest.scala | 76 +-
.../planner/expressions/SqlExpressionTest.scala | 113 +-
.../planner/expressions/TemporalTypesTest.scala | 638 ++----
.../expressions/utils/ArrayTypeTestBase.scala | 21 +-
.../expressions/utils/CompositeTypeTestBase.scala | 22 +-
.../expressions/utils/ExpressionTestBase.scala | 82 +-
.../expressions/utils/MapTypeTestBase.scala | 21 +-
.../expressions/utils/RowTypeTestBase.scala | 7 +-
.../utils/ScalarOperatorsTestBase.scala | 60 +-
.../expressions/utils/ScalarTypesTestBase.scala | 21 +-
.../utils/userDefinedScalarFunctions.scala | 6 +-
.../validation/ArrayTypeValidationTest.scala | 10 +-
.../validation/CompositeAccessValidationTest.scala | 3 -
.../validation/MapTypeValidationTest.scala | 1 -
.../validation/RowTypeValidationTest.scala | 1 -
.../validation/ScalarFunctionsValidationTest.scala | 50 +-
.../validation/ScalarOperatorsValidationTest.scala | 19 +-
.../utils/TestCollectionTableFactory.scala | 77 +-
.../planner/match/PatternTranslatorTest.scala | 94 +-
.../planner/match/PatternTranslatorTestBase.scala | 38 +-
.../table/planner/plan/batch/sql/CalcTest.scala | 6 +-
.../plan/batch/sql/CompactManagedTableTest.scala | 5 +-
.../plan/batch/sql/DagOptimizationTest.scala | 39 +-
.../plan/batch/sql/DeadlockBreakupTest.scala | 128 +-
.../planner/plan/batch/sql/LegacyLimitTest.scala | 2 +-
.../planner/plan/batch/sql/LegacySinkTest.scala | 41 +-
.../plan/batch/sql/LegacyTableSourceTest.scala | 82 +-
.../table/planner/plan/batch/sql/LimitTest.scala | 31 +-
.../plan/batch/sql/MultipleInputCreationTest.scala | 93 +-
.../plan/batch/sql/PartitionableSinkTest.scala | 1 -
.../plan/batch/sql/PartitionableSourceTest.scala | 46 +-
.../table/planner/plan/batch/sql/RankTest.scala | 44 +-
.../plan/batch/sql/RemoveCollationTest.scala | 110 +-
.../planner/plan/batch/sql/RemoveShuffleTest.scala | 217 +-
.../planner/plan/batch/sql/SetOperatorsTest.scala | 11 +-
.../planner/plan/batch/sql/SortLimitTest.scala | 5 +-
.../table/planner/plan/batch/sql/SortTest.scala | 27 +-
.../planner/plan/batch/sql/SubplanReuseTest.scala | 144 +-
.../planner/plan/batch/sql/TableScanTest.scala | 200 +-
.../planner/plan/batch/sql/TableSinkTest.scala | 145 +-
.../planner/plan/batch/sql/TableSourceTest.scala | 132 +-
.../table/planner/plan/batch/sql/UnionTest.scala | 1 -
.../table/planner/plan/batch/sql/ValuesTest.scala | 1 -
.../plan/batch/sql/WindowTableFunctionTest.scala | 33 +-
.../sql/agg/AggregateReduceGroupingTest.scala | 4 +-
.../plan/batch/sql/agg/AggregateTestBase.scala | 169 +-
.../plan/batch/sql/agg/DistinctAggregateTest.scala | 5 +-
.../plan/batch/sql/agg/GroupWindowTest.scala | 49 +-
.../plan/batch/sql/agg/GroupingSetsTest.scala | 22 +-
.../plan/batch/sql/agg/HashAggregateTest.scala | 8 +-
.../plan/batch/sql/agg/OverAggregateTest.scala | 10 +-
.../plan/batch/sql/agg/SortAggregateTest.scala | 8 +-
.../batch/sql/join/BroadcastHashJoinTest.scala | 5 +-
.../sql/join/BroadcastHashSemiAntiJoinTest.scala | 5 +-
.../planner/plan/batch/sql/join/JoinTestBase.scala | 2 +-
.../plan/batch/sql/join/LookupJoinTest.scala | 87 +-
.../plan/batch/sql/join/NestedLoopJoinTest.scala | 5 +-
.../sql/join/NestedLoopSemiAntiJoinTest.scala | 5 +-
.../plan/batch/sql/join/SemiAntiJoinTest.scala | 17 +-
.../plan/batch/sql/join/SemiAntiJoinTestBase.scala | 3 +-
.../plan/batch/sql/join/ShuffledHashJoinTest.scala | 1 -
.../sql/join/ShuffledHashSemiAntiJoinTest.scala | 1 -
.../plan/batch/sql/join/SingleRowJoinTest.scala | 1 -
.../plan/batch/sql/join/SortMergeJoinTest.scala | 5 +-
.../batch/sql/join/SortMergeSemiAntiJoinTest.scala | 5 +-
.../batch/sql/join/TemporalFunctionJoinTest.scala | 30 +-
.../plan/batch/sql/join/TemporalJoinTest.scala | 94 +-
.../planner/plan/batch/table/AggregateTest.scala | 22 +-
.../table/planner/plan/batch/table/CalcTest.scala | 12 +-
.../plan/batch/table/ColumnFunctionsTest.scala | 8 +-
.../planner/plan/batch/table/CorrelateTest.scala | 32 +-
.../planner/plan/batch/table/GroupWindowTest.scala | 27 +-
.../table/planner/plan/batch/table/JoinTest.scala | 39 +-
.../plan/batch/table/PythonAggregateTest.scala | 7 +-
.../planner/plan/batch/table/PythonCalcTest.scala | 1 -
.../table/PythonGroupWindowAggregateTest.scala | 45 +-
.../table/PythonOverWindowAggregateTest.scala | 44 +-
.../plan/batch/table/SetOperatorsTest.scala | 34 +-
.../table/TemporalTableFunctionJoinTest.scala | 14 +-
.../stringexpr/AggregateStringExpressionTest.scala | 101 +-
.../stringexpr/CalcStringExpressionTest.scala | 190 +-
.../stringexpr/CorrelateStringExpressionTest.scala | 20 +-
.../stringexpr/JoinStringExpressionTest.scala | 39 +-
.../batch/table/stringexpr/SetOperatorsTest.scala | 1 -
.../stringexpr/SortStringExpressionTest.scala | 3 +-
.../table/validation/AggregateValidationTest.scala | 35 +-
.../table/validation/CalcValidationTest.scala | 24 +-
.../table/validation/CorrelateValidationTest.scala | 12 +-
.../validation/GroupWindowValidationTest.scala | 31 +-
.../table/validation/JoinValidationTest.scala | 26 +-
.../validation/OverWindowValidationTest.scala | 15 +-
.../validation/SetOperatorsValidationTest.scala | 18 +-
.../table/validation/SortValidationTest.scala | 1 -
.../common/AggregateReduceGroupingTestBase.scala | 158 +-
.../plan/common/DistinctAggregateTestBase.scala | 16 +-
.../planner/plan/common/JoinReorderTestBase.scala | 188 +-
.../planner/plan/common/PartialInsertTest.scala | 74 +-
.../planner/plan/common/TableFactoryTest.scala | 11 +-
.../table/planner/plan/common/UnnestTestBase.scala | 12 +-
.../planner/plan/common/ViewsExpandingTest.scala | 99 +-
.../table/planner/plan/cost/FlinkCostTest.scala | 36 +-
.../table/planner/plan/hint/OptionsHintTest.scala | 79 +-
.../metadata/AggCallSelectivityEstimatorTest.scala | 373 ++-
.../metadata/FlinkRelMdColumnIntervalTest.scala | 365 +--
.../metadata/FlinkRelMdColumnNullCountTest.scala | 147 +-
.../FlinkRelMdColumnOriginNullCountTest.scala | 68 +-
.../metadata/FlinkRelMdColumnUniquenessTest.scala | 467 ++--
.../metadata/FlinkRelMdCumulativeCostTest.scala | 29 +-
.../metadata/FlinkRelMdDistinctRowCountTest.scala | 681 +++---
.../plan/metadata/FlinkRelMdDistributionTest.scala | 21 +-
.../FlinkRelMdFilteredColumnIntervalTest.scala | 253 ++-
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 1223 ++++++----
.../FlinkRelMdModifiedMonotonicityTest.scala | 160 +-
.../metadata/FlinkRelMdNonCumulativeCostTest.scala | 27 +-
.../FlinkRelMdPercentageOriginalRowsTest.scala | 44 +-
.../metadata/FlinkRelMdPopulationSizeTest.scala | 325 +--
.../plan/metadata/FlinkRelMdRowCollationTest.scala | 94 +-
.../plan/metadata/FlinkRelMdRowCountTest.scala | 197 +-
.../plan/metadata/FlinkRelMdSelectivityTest.scala | 536 +++--
.../planner/plan/metadata/FlinkRelMdSizeTest.scala | 181 +-
.../plan/metadata/FlinkRelMdUniqueGroupsTest.scala | 684 ++++--
.../plan/metadata/FlinkRelMdUniqueKeysTest.scala | 166 +-
.../plan/metadata/FlinkRelMdUpsertKeysTest.scala | 181 +-
.../metadata/MetadataHandlerConsistencyTest.scala | 95 +-
.../planner/plan/metadata/MetadataTestUtil.scala | 131 +-
.../plan/metadata/SelectivityEstimatorTest.scala | 794 ++++---
.../planner/plan/nodes/calcite/ExpandTest.scala | 9 +-
.../plan/nodes/calcite/RelNodeTestBase.scala | 20 +-
.../optimize/program/FlinkChainedProgramTest.scala | 36 +-
.../program/FlinkHepRuleSetProgramTest.scala | 44 +-
.../optimize/program/FlinkVolcanoProgramTest.scala | 18 +-
.../logical/AggregateReduceGroupingRuleTest.scala | 9 +-
.../logical/CalcPruneAggregateCallRuleTest.scala | 31 +-
.../CalcPythonCorrelateTransposeRuleTest.scala | 7 +-
.../rules/logical/CalcRankTransposeRuleTest.scala | 29 +-
.../rules/logical/ConvertToNotInOrInRuleTest.scala | 25 +-
.../logical/CorrelateSortToRankRuleTest.scala | 14 +-
.../logical/DecomposeGroupingSetsRuleTest.scala | 5 +-
.../logical/ExpressionReductionRulesTest.scala | 5 +-
...AggregateExpandDistinctAggregatesRuleTest.scala | 6 +-
.../FlinkAggregateJoinTransposeRuleTest.scala | 22 +-
.../logical/FlinkAggregateRemoveRuleTest.scala | 40 +-
.../rules/logical/FlinkCalcMergeRuleTest.scala | 25 +-
.../rules/logical/FlinkFilterJoinRuleTest.scala | 13 +-
.../logical/FlinkJoinPushExpressionsRuleTest.scala | 8 +-
.../logical/FlinkJoinToMultiJoinRuleTest.scala | 8 +-
.../rules/logical/FlinkLimit0RemoveRuleTest.scala | 9 +-
.../FlinkLogicalRankRuleForConstantRangeTest.scala | 5 +-
.../FlinkLogicalRankRuleForRangeEndTest.scala | 5 +-
.../rules/logical/FlinkPruneEmptyRulesTest.scala | 8 +-
.../FlinkSemiAntiJoinFilterTransposeRuleTest.scala | 12 +-
.../FlinkSemiAntiJoinJoinTransposeRuleTest.scala | 14 +-
...FlinkSemiAntiJoinProjectTransposeRuleTest.scala | 15 +-
.../JoinConditionEqualityTransferRuleTest.scala | 5 +-
.../logical/JoinConditionTypeCoerceRuleTest.scala | 8 +-
.../JoinDependentConditionDerivationRuleTest.scala | 15 +-
.../logical/JoinDeriveNullFilterRuleTest.scala | 50 +-
...lCorrelateToJoinFromTemporalTableRuleTest.scala | 108 +-
.../plan/rules/logical/LogicalUnnestRuleTest.scala | 5 +-
.../ProjectPruneAggregateCallRuleTest.scala | 19 +-
.../ProjectSemiAntiJoinTransposeRuleTest.scala | 7 +-
...ojectWindowTableFunctionTransposeRuleTest.scala | 36 +-
.../logical/PruneAggregateCallRuleTestBase.scala | 7 +-
...shFilterIntoLegacyTableSourceScanRuleTest.scala | 16 +-
...ushLimitIntoLegacyTableSourceScanRuleTest.scala | 10 +-
...artitionIntoLegacyTableSourceScanRuleTest.scala | 35 +-
...hProjectIntoLegacyTableSourceScanRuleTest.scala | 15 +-
.../rules/logical/PythonCalcSplitRuleTest.scala | 11 +-
.../logical/PythonCorrelateSplitRuleTest.scala | 7 +-
.../rules/logical/PythonMapMergeRuleTest.scala | 18 +-
.../logical/RankNumberColumnRemoveRuleTest.scala | 5 +-
.../logical/RemoveSingleAggregateRuleTest.scala | 5 +-
.../ReplaceIntersectWithSemiJoinRuleTest.scala | 5 +-
.../logical/ReplaceMinusWithAntiJoinRuleTest.scala | 5 +-
.../logical/RewriteIntersectAllRuleTest.scala | 5 +-
.../rules/logical/RewriteMinusAllRuleTest.scala | 5 +-
.../RewriteMultiJoinConditionRuleTest.scala | 58 +-
.../logical/SimplifyFilterConditionRuleTest.scala | 10 +-
.../logical/SimplifyJoinConditionRuleTest.scala | 4 +-
.../rules/logical/SplitAggregateRuleTest.scala | 20 +-
...SplitPythonConditionFromCorrelateRuleTest.scala | 11 +-
.../SplitPythonConditionFromJoinRuleTest.scala | 11 +-
.../TemporalJoinRewriteWithUniqueKeyRuleTest.scala | 161 +-
.../rules/logical/WindowGroupReorderRuleTest.scala | 4 +-
.../subquery/FlinkRewriteSubQueryRuleTest.scala | 43 +-
.../logical/subquery/SubQueryAntiJoinTest.scala | 9 +-
.../logical/subquery/SubQuerySemiJoinTest.scala | 17 +-
.../rules/logical/subquery/SubQueryTestBase.scala | 7 +-
.../SubqueryCorrelateVariablesValidationTest.scala | 34 +-
.../batch/EnforceLocalAggRuleTestBase.scala | 2 -
.../batch/EnforceLocalHashAggRuleTest.scala | 45 +-
.../batch/EnforceLocalSortAggRuleTest.scala | 47 +-
.../RemoveRedundantLocalHashAggRuleTest.scala | 23 +-
.../batch/RemoveRedundantLocalRankRuleTest.scala | 4 +-
.../RemoveRedundantLocalSortAggRuleTest.scala | 18 +-
.../stream/ChangelogModeInferenceTest.scala | 277 ++-
...xpandWindowTableFunctionTransposeRuleTest.scala | 76 +-
...signerChangelogNormalizeTransposeRuleTest.scala | 110 +-
.../plan/schema/TimeIndicatorRelDataTypeTest.scala | 5 +-
.../planner/plan/stats/ValueIntervalTest.scala | 24 +-
.../table/planner/plan/stream/sql/CalcTest.scala | 5 +-
.../plan/stream/sql/DagOptimizationTest.scala | 201 +-
.../planner/plan/stream/sql/DeduplicateTest.scala | 18 +-
.../plan/stream/sql/FilterableSourceTest.scala | 39 +-
.../planner/plan/stream/sql/LegacySinkTest.scala | 67 +-
.../plan/stream/sql/LegacyTableSourceTest.scala | 215 +-
.../table/planner/plan/stream/sql/LimitTest.scala | 8 +-
.../plan/stream/sql/LimitableSourceTest.scala | 5 +-
.../plan/stream/sql/MatchRecognizeTest.scala | 1 -
.../stream/sql/MiniBatchIntervalInferTest.scala | 236 +-
.../plan/stream/sql/ModifiedMonotonicityTest.scala | 24 +-
.../plan/stream/sql/PartitionableSinkTest.scala | 1 -
.../plan/stream/sql/PartitionableSourceTest.scala | 46 +-
.../table/planner/plan/stream/sql/RankTest.scala | 245 +-
.../stream/sql/RelTimeIndicatorConverterTest.scala | 30 +-
.../planner/plan/stream/sql/SetOperatorsTest.scala | 4 +-
.../planner/plan/stream/sql/SortLimitTest.scala | 8 +-
.../table/planner/plan/stream/sql/SortTest.scala | 5 +-
.../plan/stream/sql/SourceWatermarkTest.scala | 148 +-
.../planner/plan/stream/sql/SubplanReuseTest.scala | 84 +-
.../planner/plan/stream/sql/TableScanTest.scala | 830 ++++---
.../planner/plan/stream/sql/TableSinkTest.scala | 803 ++++---
.../planner/plan/stream/sql/TableSourceTest.scala | 3 +-
.../table/planner/plan/stream/sql/UnionTest.scala | 78 +-
.../table/planner/plan/stream/sql/ValuesTest.scala | 1 -
.../plan/stream/sql/WindowDeduplicateTest.scala | 52 +-
.../planner/plan/stream/sql/WindowRankTest.scala | 34 +-
.../plan/stream/sql/WindowTableFunctionTest.scala | 63 +-
.../plan/stream/sql/agg/AggregateTest.scala | 310 +--
.../stream/sql/agg/DistinctAggregateTest.scala | 11 +-
.../plan/stream/sql/agg/GroupWindowTest.scala | 100 +-
.../plan/stream/sql/agg/GroupingSetsTest.scala | 26 +-
.../stream/sql/agg/IncrementalAggregateTest.scala | 5 +-
.../plan/stream/sql/agg/OverAggregateTest.scala | 17 +-
.../stream/sql/agg/TwoStageAggregateTest.scala | 11 +-
.../plan/stream/sql/agg/WindowAggregateTest.scala | 183 +-
.../plan/stream/sql/join/IntervalJoinTest.scala | 98 +-
.../planner/plan/stream/sql/join/JoinTest.scala | 292 ++-
.../plan/stream/sql/join/LookupJoinTest.scala | 340 +--
.../plan/stream/sql/join/SemiAntiJoinTest.scala | 1 -
.../stream/sql/join/TemporalFunctionJoinTest.scala | 35 +-
.../plan/stream/sql/join/TemporalJoinTest.scala | 428 ++--
.../plan/stream/sql/join/WindowJoinTest.scala | 111 +-
.../planner/plan/stream/table/AggregateTest.scala | 36 +-
.../table/planner/plan/stream/table/CalcTest.scala | 36 +-
.../plan/stream/table/ColumnFunctionsTest.scala | 44 +-
.../planner/plan/stream/table/CorrelateTest.scala | 38 +-
.../table/GroupWindowTableAggregateTest.scala | 11 +-
.../plan/stream/table/GroupWindowTest.scala | 170 +-
.../table/planner/plan/stream/table/JoinTest.scala | 83 +-
.../plan/stream/table/LegacyTableSourceTest.scala | 199 +-
.../plan/stream/table/OverAggregateTest.scala | 166 +-
.../plan/stream/table/PythonAggregateTest.scala | 23 +-
.../planner/plan/stream/table/PythonCalcTest.scala | 1 -
.../table/PythonGroupWindowAggregateTest.scala | 77 +-
.../table/PythonOverWindowAggregateTest.scala | 33 +-
.../stream/table/PythonTableAggregateTest.scala | 13 +-
.../plan/stream/table/SetOperatorsTest.scala | 9 +-
.../plan/stream/table/TableAggregateTest.scala | 11 +-
.../plan/stream/table/TableSourceTest.scala | 30 +-
.../table/TemporalTableFunctionJoinTest.scala | 71 +-
.../plan/stream/table/TwoStageAggregateTest.scala | 11 +-
.../stringexpr/AggregateStringExpressionTest.scala | 59 +-
.../stringexpr/CalcStringExpressionTest.scala | 25 +-
.../stringexpr/CorrelateStringExpressionTest.scala | 32 +-
.../GroupWindowStringExpressionTest.scala | 98 +-
...pWindowTableAggregateStringExpressionTest.scala | 76 +-
.../OverWindowStringExpressionTest.scala | 109 +-
.../SetOperatorsStringExpressionTest.scala | 1 -
.../TableAggregateStringExpressionTest.scala | 10 +-
.../table/validation/AggregateValidationTest.scala | 22 +-
.../table/validation/CalcValidationTest.scala | 76 +-
.../table/validation/CorrelateValidationTest.scala | 59 +-
.../GroupWindowTableAggregateValidationTest.scala | 15 +-
.../validation/GroupWindowValidationTest.scala | 97 +-
.../validation/LegacyTableSinkValidationTest.scala | 36 +-
.../validation/OverWindowValidationTest.scala | 63 +-
.../validation/SetOperatorsValidationTest.scala | 5 +-
.../validation/TableAggregateValidationTest.scala | 29 +-
.../TemporalTableJoinValidationTest.scala | 32 +-
.../validation/UnsupportedOpsValidationTest.scala | 1 -
.../plan/trait/FlinkRelDistributionTest.scala | 36 +-
.../plan/utils/ColumnIntervalUtilTest.scala | 39 +-
.../planner/plan/utils/FlinkRelMdUtilTest.scala | 1 -
.../planner/plan/utils/FlinkRelOptUtilTest.scala | 72 +-
.../planner/plan/utils/FlinkRexUtilTest.scala | 412 ++--
.../planner/plan/utils/InputTypeBuilder.scala | 1 -
.../plan/utils/NestedProjectionUtilTest.scala | 106 +-
.../planner/plan/utils/PartitionPrunerTest.scala | 32 +-
.../planner/plan/utils/RexNodeExtractorTest.scala | 245 +-
.../planner/plan/utils/RexNodeRewriterTest.scala | 65 +-
.../table/planner/plan/utils/RexNodeTestBase.scala | 39 +-
.../plan/utils/TestContextTableFactory.scala | 14 +-
.../table/planner/plan/utils/lookupFunctions.scala | 40 +-
.../flink/table/planner/plan/utils/pojos.scala | 6 +-
.../planner/runtime/FileSystemITCaseBase.scala | 124 +-
.../batch/sql/BatchFileSystemITCaseBase.scala | 19 +-
.../planner/runtime/batch/sql/CalcITCase.scala | 1180 ++++++----
.../runtime/batch/sql/CodeSplitITCase.scala | 41 +-
.../runtime/batch/sql/CorrelateITCase.scala | 99 +-
.../runtime/batch/sql/CorrelateITCase2.scala | 66 +-
.../planner/runtime/batch/sql/DecimalITCase.scala | 76 +-
.../batch/sql/FileSystemTestCsvITCase.scala | 5 +-
.../runtime/batch/sql/LegacyLimitITCase.scala | 50 +-
.../batch/sql/LegacyTableSourceITCase.scala | 183 +-
.../runtime/batch/sql/Limit0RemoveITCase.scala | 1 -
.../planner/runtime/batch/sql/LimitITCase.scala | 3 +-
.../planner/runtime/batch/sql/MiscITCase.scala | 116 +-
.../runtime/batch/sql/MultipleInputITCase.scala | 33 +-
.../runtime/batch/sql/OverAggregateITCase.scala | 810 ++++---
.../batch/sql/PartitionableSinkITCase.scala | 199 +-
.../batch/sql/PartitionableSourceITCase.scala | 134 +-
.../planner/runtime/batch/sql/RankITCase.scala | 191 +-
.../runtime/batch/sql/SetOperatorsITCase.scala | 31 +-
.../runtime/batch/sql/SortLimitITCase.scala | 60 +-
.../runtime/batch/sql/TableScanITCase.scala | 22 +-
.../runtime/batch/sql/TableSinkITCase.scala | 51 +-
.../runtime/batch/sql/TableSourceITCase.scala | 253 ++-
.../runtime/batch/sql/TimestampITCase.scala | 204 +-
.../planner/runtime/batch/sql/UnionITCase.scala | 46 +-
.../planner/runtime/batch/sql/UnnestITCase.scala | 105 +-
.../planner/runtime/batch/sql/ValuesITCase.scala | 1 -
.../batch/sql/WindowTableFunctionITCase.scala | 1019 +++++++--
.../batch/sql/agg/AggregateITCaseBase.scala | 321 ++-
.../sql/agg/AggregateJoinTransposeITCase.scala | 115 +-
.../sql/agg/AggregateReduceGroupingITCase.scala | 597 +++--
.../batch/sql/agg/AggregateRemoveITCase.scala | 211 +-
.../sql/agg/DistinctAggregateITCaseBase.scala | 88 +-
.../runtime/batch/sql/agg/GroupWindowITCase.scala | 332 ++-
.../runtime/batch/sql/agg/GroupingSetsITCase.scala | 332 ++-
.../runtime/batch/sql/agg/HashAggITCase.scala | 11 +-
.../sql/agg/HashDistinctAggregateITCase.scala | 8 +-
.../batch/sql/agg/PruneAggregateCallITCase.scala | 7 +-
.../runtime/batch/sql/agg/SortAggITCase.scala | 97 +-
.../sql/agg/SortDistinctAggregateITCase.scala | 8 +-
.../runtime/batch/sql/join/InnerJoinITCase.scala | 9 +-
.../sql/join/JoinConditionTypeCoerceITCase.scala | 155 +-
.../runtime/batch/sql/join/JoinITCase.scala | 620 +++--
.../runtime/batch/sql/join/JoinITCaseHelper.scala | 5 +-
.../batch/sql/join/JoinWithoutKeyITCase.scala | 220 +-
.../runtime/batch/sql/join/LookupJoinITCase.scala | 132 +-
.../runtime/batch/sql/join/OuterJoinITCase.scala | 245 +-
.../runtime/batch/sql/join/ScalarQueryITCase.scala | 9 +-
.../runtime/batch/sql/join/SemiJoinITCase.scala | 81 +-
.../runtime/batch/table/AggregationITCase.scala | 212 +-
.../planner/runtime/batch/table/CalcITCase.scala | 175 +-
.../runtime/batch/table/CorrelateITCase.scala | 83 +-
.../runtime/batch/table/DecimalITCase.scala | 80 +-
.../runtime/batch/table/GroupWindowITCase.scala | 213 +-
.../planner/runtime/batch/table/JoinITCase.scala | 219 +-
.../runtime/batch/table/LegacyLimitITCase.scala | 21 +-
.../batch/table/LegacyTableSinkITCase.scala | 144 +-
.../planner/runtime/batch/table/LimitITCase.scala | 2 +-
.../runtime/batch/table/OverAggregateITCase.scala | 399 ++--
.../runtime/batch/table/SetOperatorsITCase.scala | 34 +-
.../planner/runtime/batch/table/SortITCase.scala | 29 +-
.../runtime/batch/table/TableSinkITCase.scala | 529 +++--
...AbstractTwoInputStreamOperatorWithTTLTest.scala | 26 +-
.../harness/GroupAggregateHarnessTest.scala | 16 +-
.../planner/runtime/harness/HarnessTestBase.scala | 42 +-
.../runtime/harness/OverAggregateHarnessTest.scala | 762 +++----
.../planner/runtime/harness/RankHarnessTest.scala | 21 +-
.../harness/TableAggregateHarnessTest.scala | 29 +-
.../harness/WindowAggregateHarnessTest.scala | 426 +++-
...WindowAggregateUseDaylightTimeHarnessTest.scala | 97 +-
.../harness/WindowTableFunctionHarnessTest.scala | 881 +++++---
.../runtime/stream/FsStreamingSinkITCaseBase.scala | 86 +-
.../runtime/stream/sql/AggregateITCase.scala | 631 +++---
.../runtime/stream/sql/AggregateRemoveITCase.scala | 226 +-
.../runtime/stream/sql/AsyncLookupJoinITCase.scala | 112 +-
.../planner/runtime/stream/sql/CalcITCase.scala | 249 +-
.../runtime/stream/sql/ChangelogSourceITCase.scala | 296 ++-
.../runtime/stream/sql/CorrelateITCase.scala | 100 +-
.../runtime/stream/sql/DataStreamScalaITCase.scala | 31 +-
.../runtime/stream/sql/DeduplicateITCase.scala | 136 +-
.../stream/sql/FilterableSourceITCase.scala | 18 +-
.../stream/sql/FsStreamingSinkTestCsvITCase.scala | 5 +-
.../runtime/stream/sql/GroupWindowITCase.scala | 165 +-
.../runtime/stream/sql/IntervalJoinITCase.scala | 176 +-
.../planner/runtime/stream/sql/JoinITCase.scala | 397 +++-
.../stream/sql/LegacyTableSourceITCase.scala | 195 +-
.../runtime/stream/sql/Limit0RemoveITCase.scala | 3 +-
.../planner/runtime/stream/sql/LimitITCase.scala | 15 +-
.../runtime/stream/sql/LimitableSourceITCase.scala | 19 +-
.../runtime/stream/sql/LookupJoinITCase.scala | 176 +-
.../runtime/stream/sql/MatchRecognizeITCase.scala | 112 +-
.../runtime/stream/sql/OverAggregateITCase.scala | 439 ++--
.../stream/sql/PartitionableSourceITCase.scala | 48 +-
.../stream/sql/PruneAggregateCallITCase.scala | 4 +-
.../planner/runtime/stream/sql/RankITCase.scala | 430 ++--
.../stream/sql/SemiAntiJoinStreamITCase.scala | 75 +-
.../runtime/stream/sql/SetOperatorsITCase.scala | 13 +-
.../planner/runtime/stream/sql/SortITCase.scala | 41 +-
.../runtime/stream/sql/SortLimitITCase.scala | 25 +-
.../runtime/stream/sql/SourceWatermarkITCase.scala | 71 +-
.../runtime/stream/sql/SplitAggregateITCase.scala | 312 +--
.../stream/sql/StreamFileSystemITCaseBase.scala | 24 +-
.../stream/sql/StreamFileSystemTestCsvITCase.scala | 5 +-
.../stream/sql/StreamTableEnvironmentITCase.scala | 93 +-
.../runtime/stream/sql/TableScanITCase.scala | 31 +-
.../runtime/stream/sql/TableSinkITCase.scala | 230 +-
.../runtime/stream/sql/TableSourceITCase.scala | 120 +-
.../runtime/stream/sql/TemporalJoinITCase.scala | 520 ++---
.../runtime/stream/sql/TemporalSortITCase.scala | 88 +-
.../sql/TemporalTableFunctionJoinITCase.scala | 40 +-
.../runtime/stream/sql/TimeAttributeITCase.scala | 75 +-
.../runtime/stream/sql/TimestampITCase.scala | 80 +-
.../planner/runtime/stream/sql/UnnestITCase.scala | 53 +-
.../planner/runtime/stream/sql/ValuesITCase.scala | 5 +-
.../runtime/stream/sql/WindowAggregateITCase.scala | 92 +-
.../stream/sql/WindowDeduplicateITCase.scala | 52 +-
.../stream/sql/WindowDistinctAggregateITCase.scala | 191 +-
.../runtime/stream/sql/WindowJoinITCase.scala | 88 +-
.../runtime/stream/sql/WindowRankITCase.scala | 91 +-
.../stream/sql/WindowTableFunctionITCase.scala | 60 +-
.../runtime/stream/table/AggregateITCase.scala | 132 +-
.../planner/runtime/stream/table/CalcITCase.scala | 228 +-
.../runtime/stream/table/CorrelateITCase.scala | 77 +-
.../runtime/stream/table/GroupWindowITCase.scala | 101 +-
.../table/GroupWindowTableAggregateITCase.scala | 60 +-
.../planner/runtime/stream/table/JoinITCase.scala | 611 +++--
.../stream/table/LegacyTableSinkITCase.scala | 295 ++-
.../runtime/stream/table/OverAggregateITCase.scala | 282 ++-
.../runtime/stream/table/RetractionITCase.scala | 44 +-
.../runtime/stream/table/SetOperatorsITCase.scala | 37 +-
.../runtime/stream/table/SubQueryITCase.scala | 26 +-
.../stream/table/TableAggregateITCase.scala | 95 +-
.../runtime/stream/table/TableSinkITCase.scala | 1377 +++++------
.../stream/table/TableToDataStreamITCase.scala | 139 +-
.../planner/runtime/utils/BatchTableEnvUtil.scala | 305 ++-
.../planner/runtime/utils/BatchTestBase.scala | 262 ++-
.../runtime/utils/CollectionBatchExecTable.scala | 43 +-
.../utils/InMemoryLookupableTableSource.scala | 65 +-
.../planner/runtime/utils/SortTestUtils.scala | 52 +-
.../planner/runtime/utils/StreamTableEnvUtil.scala | 17 +-
.../planner/runtime/utils/StreamTestSink.scala | 90 +-
.../planner/runtime/utils/StreamingTestBase.scala | 13 +-
.../runtime/utils/StreamingWithAggTestBase.scala | 10 +-
.../utils/StreamingWithMiniBatchTestBase.scala | 4 +-
.../runtime/utils/StreamingWithStateTestBase.scala | 32 +-
.../table/planner/runtime/utils/TestData.scala | 660 ++++--
.../table/planner/runtime/utils/TestSinkUtil.scala | 11 +-
.../table/planner/runtime/utils/TimeTestUtil.scala | 17 +-
.../utils/UserDefinedFunctionTestUtils.scala | 36 +-
.../flink/table/planner/utils/AvgAggFunction.scala | 117 +-
.../table/planner/utils/CountAggFunction.scala | 6 +-
.../table/planner/utils/DateTimeTestUtil.scala | 1 -
.../planner/utils/LogicalPlanFormatUtils.scala | 10 +-
.../planner/utils/MemoryTableSourceSinkUtil.scala | 34 +-
.../flink/table/planner/utils/SumAggFunction.scala | 44 +-
.../flink/table/planner/utils/TableTestBase.scala | 692 +++---
.../utils/TestLegacyLimitableTableSource.scala | 24 +-
.../planner/utils/UserDefinedAggFunctions.scala | 27 +-
.../utils/UserDefinedTableAggFunctions.scala | 26 +-
.../planner/utils/UserDefinedTableFunctions.scala | 40 +-
.../table/planner/utils/testTableSourceSinks.scala | 401 ++--
flink-tests/pom.xml | 26 +-
.../api/scala/actions/CountCollectITCase.scala | 17 +-
.../PartitionOperatorTranslationTest.scala | 33 +-
.../BatchScalaAPICompletenessTest.scala | 252 ++-
.../ScalaAPICompletenessTestBase.scala | 29 +-
.../api/scala/functions/ClosureCleanerITCase.scala | 19 +-
.../scala/io/ScalaCsvReaderWithPOJOITCase.scala | 12 +-
.../manual/MassiveCaseClassSortingITCase.scala | 129 +-
.../api/scala/migration/MigrationTestTypes.scala | 1 -
.../migration/ScalaSerializersMigrationTest.scala | 12 +-
.../StatefulJobSavepointMigrationITCase.scala | 81 +-
...StatefulJobWBroadcastStateMigrationITCase.scala | 124 +-
.../api/scala/operators/AggregateITCase.scala | 17 +-
.../scala/operators/AggregateOperatorTest.scala | 13 +-
.../flink/api/scala/operators/CoGroupITCase.scala | 235 +-
.../api/scala/operators/CoGroupOperatorTest.scala | 61 +-
.../flink/api/scala/operators/CrossITCase.scala | 64 +-
.../flink/api/scala/operators/DistinctITCase.scala | 18 +-
.../api/scala/operators/DistinctOperatorTest.scala | 18 +-
.../flink/api/scala/operators/ExamplesITCase.scala | 23 +-
.../flink/api/scala/operators/FilterITCase.scala | 38 +-
.../flink/api/scala/operators/FirstNITCase.scala | 17 +-
.../api/scala/operators/FirstNOperatorTest.scala | 61 +-
.../flink/api/scala/operators/FlatMapITCase.scala | 71 +-
.../api/scala/operators/GroupCombineITCase.scala | 24 +-
.../api/scala/operators/GroupReduceITCase.scala | 761 +++----
.../flink/api/scala/operators/GroupingTest.scala | 34 +-
.../flink/api/scala/operators/JoinITCase.scala | 88 +-
.../api/scala/operators/JoinOperatorTest.scala | 36 +-
.../flink/api/scala/operators/MapITCase.scala | 32 +-
.../api/scala/operators/OuterJoinITCase.scala | 83 +-
.../api/scala/operators/PartitionITCase.scala | 84 +-
.../flink/api/scala/operators/ReduceITCase.scala | 89 +-
.../flink/api/scala/operators/SampleITCase.scala | 9 +-
.../api/scala/operators/SortPartitionITCase.scala | 62 +-
.../api/scala/operators/SumMinMaxITCase.scala | 21 +-
.../flink/api/scala/operators/UnionITCase.scala | 13 +-
.../translation/AggregateTranslationTest.scala | 16 +-
.../CoGroupCustomPartitioningTest.scala | 167 +-
.../CoGroupGroupSortTranslationTest.scala | 107 +-
...CustomPartitioningGroupingKeySelectorTest.scala | 102 +-
.../CustomPartitioningGroupingPojoTest.scala | 80 +-
.../CustomPartitioningGroupingTupleTest.scala | 184 +-
.../translation/CustomPartitioningTest.scala | 143 +-
.../DeltaIterationTranslationTest.scala | 52 +-
.../translation/JoinCustomPartitioningTest.scala | 167 +-
.../translation/PartitioningTestClasses.scala | 3 -
.../translation/ReduceTranslationTest.scala | 63 +-
.../CaseClassNormalizedKeySortingTest.scala | 69 +-
.../scala/runtime/ScalaSpecialTypesITCase.scala | 101 +-
.../flink/api/scala/util/CollectionDataSets.scala | 56 +-
.../flink/api/scala/util/DataSetUtilsITCase.scala | 7 +-
flink-yarn-tests/pom.xml | 9 -
pom.xml | 59 +-
tools/ci/compile.sh | 6 +-
tools/maven/scalastyle-config.xml | 146 --
1340 files changed, 61912 insertions(+), 56725 deletions(-)
create mode 100644 .scalafmt.conf
delete mode 100644 tools/maven/scalastyle-config.xml
[flink] 03/05: [FLINK-27317][build] Use absolute paths to .scalafmt.conf
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 09293d618bb1fafe99acf305c294a4e46a6777e9
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Apr 20 10:59:04 2022 +0200
[FLINK-27317][build] Use absolute paths to .scalafmt.conf
---
flink-connectors/flink-hadoop-compatibility/pom.xml | 2 +-
flink-connectors/flink-hcatalog/pom.xml | 2 +-
flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml | 2 +-
flink-end-to-end-tests/flink-quickstart-test/pom.xml | 2 +-
flink-examples/flink-examples-batch/pom.xml | 2 +-
flink-examples/flink-examples-streaming/pom.xml | 2 +-
flink-examples/flink-examples-table/pom.xml | 2 +-
flink-libraries/flink-cep-scala/pom.xml | 2 +-
flink-libraries/flink-gelly-examples/pom.xml | 2 +-
flink-libraries/flink-gelly-scala/pom.xml | 2 +-
flink-scala/pom.xml | 2 +-
flink-streaming-scala/pom.xml | 2 +-
flink-table/flink-table-api-scala-bridge/pom.xml | 2 +-
flink-table/flink-table-api-scala/pom.xml | 2 +-
flink-table/flink-table-planner/pom.xml | 2 +-
flink-tests/pom.xml | 2 +-
16 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 9186f05a5f7..976d7b20211 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -107,7 +107,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
index b98817870ee..23aae868c21 100644
--- a/flink-connectors/flink-hcatalog/pom.xml
+++ b/flink-connectors/flink-hcatalog/pom.xml
@@ -104,7 +104,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
index 2f32bb30b07..7402938ba4b 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
@@ -89,7 +89,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-end-to-end-tests/flink-quickstart-test/pom.xml b/flink-end-to-end-tests/flink-quickstart-test/pom.xml
index 1a2e8b6af0f..df181c9d12f 100644
--- a/flink-end-to-end-tests/flink-quickstart-test/pom.xml
+++ b/flink-end-to-end-tests/flink-quickstart-test/pom.xml
@@ -70,7 +70,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml
index 10af26aa70f..ff774111d0a 100644
--- a/flink-examples/flink-examples-batch/pom.xml
+++ b/flink-examples/flink-examples-batch/pom.xml
@@ -62,7 +62,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index aaee6c1efa8..40c1e2ed546 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -131,7 +131,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
index cc126ff93f9..cb424a1cc4f 100644
--- a/flink-examples/flink-examples-table/pom.xml
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -104,7 +104,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
index e35b50f0d69..e25d7332ee4 100644
--- a/flink-libraries/flink-cep-scala/pom.xml
+++ b/flink-libraries/flink-cep-scala/pom.xml
@@ -116,7 +116,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index 72345f9534f..e3cf09be02e 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -128,7 +128,7 @@
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index e930bc4e45e..cf660be8f47 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -103,7 +103,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 418b33399e7..9412855ef6e 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -146,7 +146,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../.scalafmt.conf</file>
+ <file>${project.basedir}/../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index 5126519ea7d..eec2636a3e4 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -142,7 +142,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../.scalafmt.conf</file>
+ <file>${project.basedir}/../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-table/flink-table-api-scala-bridge/pom.xml b/flink-table/flink-table-api-scala-bridge/pom.xml
index 4a31d9f1a94..b6b90487b18 100644
--- a/flink-table/flink-table-api-scala-bridge/pom.xml
+++ b/flink-table/flink-table-api-scala-bridge/pom.xml
@@ -84,7 +84,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-table/flink-table-api-scala/pom.xml b/flink-table/flink-table-api-scala/pom.xml
index b2a9248d651..7933d39ff8a 100644
--- a/flink-table/flink-table-api-scala/pom.xml
+++ b/flink-table/flink-table-api-scala/pom.xml
@@ -95,7 +95,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index d16df6271df..c613193dc0d 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -287,7 +287,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../../.scalafmt.conf</file>
+ <file>${project.basedir}/../../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 421ff9113ae..7e802a28d33 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -286,7 +286,7 @@ under the License.
<scala>
<scalafmt>
<version>${spotless.scalafmt.version}</version>
- <file>../.scalafmt.conf</file>
+ <file>${project.basedir}/../.scalafmt.conf</file>
</scalafmt>
<licenseHeader>
<content>${spotless.license.header}</content>
[flink] 05/05: [FLINK-27534][build] Format code with Spotless/scalafmt
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 5d5d4aee5079e498435dc9afe22b1d6514dbda2b
Author: Timo Walther <tw...@apache.org>
AuthorDate: Mon May 16 14:49:42 2022 +0200
[FLINK-27534][build] Format code with Spotless/scalafmt
---
.../scala/hadoop/mapred/HadoopInputFormat.scala | 1 +
.../scala/hadoop/mapred/HadoopOutputFormat.scala | 1 +
.../scala/hadoop/mapreduce/HadoopInputFormat.scala | 2 +-
.../hadoop/mapreduce/HadoopOutputFormat.scala | 2 +-
.../hadoopcompatibility/scala/HadoopInputs.scala | 78 +-
.../scala/ScalaWritableTypeInfoTest.scala | 1 +
.../scala/WordCountMapredITCase.scala | 23 +-
.../scala/WordCountMapreduceITCase.scala | 29 +-
.../flink/hcatalog/scala/HCatInputFormat.scala | 444 ++--
.../org/apache/flink/tests/scala/ScalaJob.scala | 16 +-
.../test/Elasticsearch7SinkExample.scala | 39 +-
.../flink/examples/scala/clustering/KMeans.scala | 89 +-
.../examples/scala/graph/ConnectedComponents.scala | 81 +-
.../flink/examples/scala/graph/DeltaPageRank.scala | 104 +-
.../flink/examples/scala/graph/EnumTriangles.scala | 85 +-
.../flink/examples/scala/graph/PageRankBasic.scala | 72 +-
.../scala/graph/TransitiveClosureNaive.scala | 67 +-
.../flink/examples/scala/misc/PiEstimation.scala | 15 +-
.../examples/scala/relational/TPCHQuery10.scala | 176 +-
.../examples/scala/relational/TPCHQuery3.scala | 129 +-
.../examples/scala/relational/WebLogAnalysis.scala | 69 +-
.../flink/examples/scala/wordcount/WordCount.scala | 13 +-
.../scala/examples/async/AsyncClient.scala | 5 +-
.../scala/examples/async/AsyncIOExample.scala | 5 +-
.../scala/examples/iteration/IterateExample.scala | 72 +-
.../streaming/scala/examples/join/WindowJoin.scala | 22 +-
.../scala/examples/join/WindowJoinSampleData.scala | 31 +-
.../examples/socket/SocketWindowWordCount.scala | 31 +-
.../GroupedProcessingTimeWindowExample.scala | 11 +-
.../examples/windowing/SessionWindowing.scala | 39 +-
.../examples/windowing/TopSpeedWindowing.scala | 54 +-
.../scala/examples/windowing/WindowWordCount.scala | 44 +-
.../scala/examples/windowing/util/CarSource.scala | 9 +-
.../scala/examples/wordcount/WordCount.scala | 56 +-
.../scala/examples/wordcount/util/CLI.scala | 1 -
.../scala/examples/StreamingExamplesITCase.scala | 98 +-
.../scala/basics/GettingStartedExample.scala | 99 +-
.../examples/scala/basics/StreamSQLExample.scala | 21 +-
.../examples/scala/basics/StreamTableExample.scala | 29 +-
.../scala/basics/WordCountSQLExample.scala | 25 +-
.../scala/org/apache/flink/cep/scala/CEP.scala | 59 +-
.../org/apache/flink/cep/scala/PatternStream.scala | 590 ++---
.../flink/cep/scala/conditions/Context.scala | 16 +-
.../scala/org/apache/flink/cep/scala/package.scala | 24 +-
.../flink/cep/scala/pattern/GroupPattern.scala | 47 +-
.../apache/flink/cep/scala/pattern/Pattern.scala | 627 +++---
.../apache/flink/cep/scala/pattern/package.scala | 36 +-
.../cep/scala/CEPScalaAPICompletenessTest.scala | 13 +-
...ernStreamScalaJavaAPIInteroperabilityTest.scala | 64 +-
.../pattern/PatternScalaAPICompletenessTest.scala | 9 +-
.../flink/cep/scala/pattern/PatternTest.scala | 111 +-
.../graph/scala/examples/ConnectedComponents.scala | 101 +-
.../examples/GSASingleSourceShortestPaths.scala | 61 +-
.../scala/examples/SingleSourceShortestPaths.scala | 71 +-
.../apache/flink/graph/scala/EdgesFunction.scala | 9 +-
.../graph/scala/EdgesFunctionWithVertexValue.scala | 11 +-
.../scala/org/apache/flink/graph/scala/Graph.scala | 1428 ++++++------
.../flink/graph/scala/NeighborsFunction.scala | 14 +-
.../scala/NeighborsFunctionWithVertexValue.scala | 23 +-
.../org/apache/flink/graph/scala/package.scala | 10 +-
.../flink/graph/scala/utils/EdgeToTuple3Map.scala | 1 -
.../graph/scala/utils/Tuple2ToVertexMap.scala | 1 -
.../flink/graph/scala/utils/Tuple3ToEdgeMap.scala | 1 -
.../graph/scala/utils/VertexToTuple2Map.scala | 1 -
.../scala/test/GellyScalaAPICompletenessTest.scala | 11 +-
.../flink/graph/scala/test/TestGraphUtils.scala | 53 +-
.../scala/test/operations/DegreesITCase.scala | 30 +-
.../operations/GraphCreationWithCsvITCase.scala | 119 +-
.../test/operations/GraphMutationsITCase.scala | 173 +-
.../test/operations/GraphOperationsITCase.scala | 217 +-
.../test/operations/JoinWithEdgesITCase.scala | 92 +-
.../test/operations/JoinWithVerticesITCase.scala | 31 +-
.../scala/test/operations/MapEdgesITCase.scala | 26 +-
.../scala/test/operations/MapVerticesITCase.scala | 23 +-
.../operations/ReduceOnEdgesMethodsITCase.scala | 88 +-
.../operations/ReduceOnNeighborMethodsITCase.scala | 67 +-
.../apache/flink/api/scala/AggregateDataSet.scala | 34 +-
.../apache/flink/api/scala/ClosureCleaner.scala | 197 +-
.../apache/flink/api/scala/CoGroupDataSet.scala | 93 +-
.../org/apache/flink/api/scala/CrossDataSet.scala | 25 +-
.../scala/org/apache/flink/api/scala/DataSet.scala | 1267 +++++------
.../flink/api/scala/ExecutionEnvironment.scala | 495 ++--
.../apache/flink/api/scala/GroupedDataSet.scala | 311 ++-
.../flink/api/scala/PartitionSortedDataSet.scala | 24 +-
.../flink/api/scala/SelectByMaxFunction.scala | 35 +-
.../flink/api/scala/SelectByMinFunction.scala | 31 +-
.../api/scala/UnfinishedCoGroupOperation.scala | 33 +-
.../api/scala/codegen/MacroContextHolder.scala | 9 +-
.../apache/flink/api/scala/codegen/TreeGen.scala | 381 ++--
.../flink/api/scala/codegen/TypeAnalyzer.scala | 214 +-
.../flink/api/scala/codegen/TypeDescriptors.scala | 34 +-
.../api/scala/codegen/TypeInformationGen.scala | 81 +-
.../acceptPartialFunctions/OnCoGroupDataSet.scala | 39 +-
.../acceptPartialFunctions/OnCrossDataSet.scala | 34 +-
.../impl/acceptPartialFunctions/OnDataSet.scala | 127 +-
.../acceptPartialFunctions/OnGroupedDataSet.scala | 86 +-
.../OnHalfUnfinishedKeyPairOperation.scala | 38 +-
.../OnJoinFunctionAssigner.scala | 34 +-
.../OnUnfinishedKeyPairOperation.scala | 39 +-
.../flink/api/scala/extensions/package.scala | 75 +-
.../org/apache/flink/api/scala/joinDataSet.scala | 108 +-
.../flink/api/scala/metrics/ScalaGauge.scala | 5 +-
.../scala/org/apache/flink/api/scala/package.scala | 58 +-
.../api/scala/typeutils/CaseClassComparator.scala | 25 +-
.../api/scala/typeutils/CaseClassSerializer.scala | 19 +-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 91 +-
.../api/scala/typeutils/EitherSerializer.scala | 8 +-
.../flink/api/scala/typeutils/EitherTypeInfo.scala | 6 +-
.../api/scala/typeutils/EnumValueComparator.scala | 6 +-
.../api/scala/typeutils/EnumValueSerializer.scala | 27 +-
.../api/scala/typeutils/EnumValueTypeInfo.scala | 14 +-
.../api/scala/typeutils/NothingSerializer.scala | 11 +-
.../api/scala/typeutils/OptionSerializer.scala | 15 +-
.../api/scala/typeutils/OptionTypeComparator.scala | 16 +-
.../flink/api/scala/typeutils/OptionTypeInfo.scala | 10 +-
.../scala/typeutils/ScalaCaseClassSerializer.scala | 23 +-
.../typeutils/ScalaEnumSerializerSnapshot.scala | 13 +-
.../api/scala/typeutils/ScalaNothingTypeInfo.scala | 2 +-
.../scala/typeutils/TraversableSerializer.scala | 55 +-
.../api/scala/typeutils/TraversableTypeInfo.scala | 6 +-
.../flink/api/scala/typeutils/TrySerializer.scala | 21 +-
.../flink/api/scala/typeutils/TryTypeInfo.scala | 10 +-
.../flink/api/scala/typeutils/TypeUtils.scala | 4 +-
.../apache/flink/api/scala/typeutils/Types.scala | 477 ++--
.../flink/api/scala/typeutils/UnitSerializer.scala | 15 +-
.../flink/api/scala/typeutils/UnitTypeInfo.scala | 2 +-
.../api/scala/unfinishedKeyPairOperation.scala | 98 +-
.../org/apache/flink/api/scala/utils/package.scala | 117 +-
.../runtime/types/FlinkScalaKryoInstantiator.scala | 30 +-
.../types/JavaIterableWrapperSerializer.scala | 10 +-
.../api/scala/DeltaIterationSanityCheckTest.scala | 94 +-
.../apache/flink/api/scala/MaxByOperatorTest.scala | 54 +-
.../apache/flink/api/scala/MinByOperatorTest.scala | 55 +-
.../flink/api/scala/SelectByFunctionTest.scala | 250 +-
.../scala/extensions/base/AcceptPFTestBase.scala | 7 +-
.../api/scala/extensions/data/KeyValuePair.scala | 14 +-
.../OnCoGroupDataSetTest.scala | 31 +-
.../OnCrossDataSetTest.scala | 11 +-
.../acceptPartialFunctions/OnDataSetTest.scala | 91 +-
.../OnGroupedDataSetTest.scala | 37 +-
.../OnHalfUnfinishedKeyPairOperationTest.scala | 90 +-
.../OnJoinFunctionAssignerTest.scala | 121 +-
.../OnUnfinishedKeyPairOperationTest.scala | 95 +-
.../SemanticPropertiesTranslationTest.scala | 65 +-
.../api/scala/io/CollectionInputFormatTest.scala | 79 +-
.../flink/api/scala/io/CsvInputFormatTest.scala | 92 +-
.../flink/api/scala/metrics/ScalaGaugeTest.scala | 1 -
.../scala/runtime/EnumValueComparatorTest.scala | 1 -
.../scala/runtime/GenericPairComparatorTest.scala | 17 +-
.../runtime/KryoGenericTypeSerializerTest.scala | 50 +-
.../runtime/ScalaSpecialTypesSerializerTest.scala | 44 +-
.../scala/runtime/TraversableSerializerTest.scala | 45 +-
.../scala/runtime/TupleComparatorILD2Test.scala | 5 +-
.../scala/runtime/TupleComparatorILD3Test.scala | 2 -
.../scala/runtime/TupleComparatorILDC3Test.scala | 4 +-
.../scala/runtime/TupleComparatorILDX1Test.scala | 6 +-
.../scala/runtime/TupleComparatorILDXC2Test.scala | 6 +-
.../scala/runtime/TupleComparatorISD1Test.scala | 3 +-
.../scala/runtime/TupleComparatorISD2Test.scala | 1 -
.../scala/runtime/TupleComparatorISD3Test.scala | 3 +-
.../runtime/TupleSerializerCompatibilityTest.scala | 17 +-
...TupleSerializerCompatibilityTestGenerator.scala | 29 +-
.../api/scala/runtime/TupleSerializerTest.scala | 113 +-
.../runtime/TupleSerializerTestInstance.scala | 22 +-
.../tuple/base/PairComparatorTestBase.scala | 13 +-
.../tuple/base/TupleComparatorTestBase.scala | 6 +-
.../api/scala/types/TypeInformationGenTest.scala | 180 +-
.../scala/typeutils/CaseClassTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/EitherSerializerTest.scala | 11 +-
.../api/scala/typeutils/EitherTypeInfoTest.scala | 9 +-
.../EnumValueSerializerCompatibilityTest.scala | 57 +-
.../scala/typeutils/EnumValueSerializerTest.scala | 10 +-
.../typeutils/EnumValueSerializerUpgradeTest.scala | 41 +-
.../scala/typeutils/EnumValueTypeInfoTest.scala | 9 +-
.../scala/typeutils/InstantiationUtilTest.scala | 17 +-
.../api/scala/typeutils/OptionTypeInfoTest.scala | 6 +-
.../ScalaCaseClassSerializerReflectionTest.scala | 10 +-
.../typeutils/ScalaCaseClassSerializerTest.scala | 8 +-
.../ScalaCaseClassSerializerUpgradeTest.scala | 39 +-
.../scala/typeutils/ScalaNothingTypeInfoTest.scala | 5 +-
.../TraversableSerializerUpgradeTest.scala | 223 +-
.../scala/typeutils/TraversableTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/TryTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/TypeExtractionTest.scala | 7 +-
.../api/scala/typeutils/TypeInfoFactoryTest.scala | 17 +-
.../api/scala/typeutils/UnitTypeInfoTest.scala | 5 +-
.../streaming/api/scala/AllWindowedStream.scala | 398 ++--
.../streaming/api/scala/AsyncDataStream.scala | 366 +--
.../api/scala/BroadcastConnectedStream.scala | 60 +-
.../streaming/api/scala/CloseableIterator.scala | 17 +-
.../streaming/api/scala/CoGroupedStreams.scala | 104 +-
.../streaming/api/scala/ConnectedStreams.scala | 266 ++-
.../flink/streaming/api/scala/DataStream.scala | 899 ++++----
.../streaming/api/scala/DataStreamUtils.scala | 58 +-
.../flink/streaming/api/scala/JoinedStreams.scala | 127 +-
.../flink/streaming/api/scala/KeyedStream.scala | 532 ++---
.../flink/streaming/api/scala/OutputTag.scala | 11 +-
.../api/scala/StreamExecutionEnvironment.scala | 1000 ++++----
.../flink/streaming/api/scala/WindowedStream.scala | 401 ++--
.../streaming/api/scala/async/AsyncFunction.scala | 56 +-
.../api/scala/async/JavaResultFutureWrapper.scala | 17 +-
.../streaming/api/scala/async/ResultFuture.scala | 40 +-
.../api/scala/async/RichAsyncFunction.scala | 31 +-
.../async/ScalaRichAsyncFunctionWrapper.scala | 15 +-
.../acceptPartialFunctions/OnConnectedStream.scala | 89 +-
.../impl/acceptPartialFunctions/OnDataStream.scala | 75 +-
.../acceptPartialFunctions/OnJoinedStream.scala | 39 +-
.../acceptPartialFunctions/OnKeyedStream.scala | 29 +-
.../acceptPartialFunctions/OnWindowedStream.scala | 49 +-
.../streaming/api/scala/extensions/package.scala | 77 +-
.../api/scala/function/AllWindowFunction.scala | 28 +-
.../scala/function/ProcessAllWindowFunction.scala | 75 +-
.../api/scala/function/ProcessWindowFunction.scala | 88 +-
.../api/scala/function/RichAllWindowFunction.scala | 14 +-
.../function/RichProcessAllWindowFunction.scala | 22 +-
.../scala/function/RichProcessWindowFunction.scala | 25 +-
.../api/scala/function/RichWindowFunction.scala | 21 +-
.../api/scala/function/StatefulFunction.scala | 12 +-
.../api/scala/function/WindowFunction.scala | 40 +-
.../function/util/ScalaAllWindowFunction.scala | 11 +-
.../util/ScalaAllWindowFunctionWrapper.scala | 8 +-
.../util/ScalaProcessWindowFunctionWrapper.scala | 44 +-
.../scala/function/util/ScalaReduceFunction.scala | 9 +-
.../scala/function/util/ScalaWindowFunction.scala | 11 +-
.../function/util/ScalaWindowFunctionWrapper.scala | 10 +-
.../apache/flink/streaming/api/scala/package.scala | 47 +-
.../api/scala/AllWindowTranslationTest.scala | 343 ++-
.../api/scala/AsyncDataStreamITCase.scala | 45 +-
.../streaming/api/scala/BroadcastStateITCase.scala | 17 +-
.../api/scala/CaseClassFieldAccessorTest.scala | 2 +-
.../streaming/api/scala/CoGroupJoinITCase.scala | 201 +-
.../streaming/api/scala/CoGroupedStreamsTest.scala | 9 +-
.../flink/streaming/api/scala/DataStreamTest.scala | 184 +-
.../streaming/api/scala/IntervalJoinITCase.scala | 22 +-
.../streaming/api/scala/JoinedStreamsTest.scala | 9 +-
.../api/scala/OutputFormatTestPrograms.scala | 40 +-
.../ReinterpretDataStreamAsKeyedStreamITCase.scala | 8 +-
.../streaming/api/scala/SideOutputITCase.scala | 124 +-
.../streaming/api/scala/SlotAllocationTest.scala | 38 +-
.../streaming/api/scala/StateTestPrograms.scala | 85 +-
.../api/scala/StreamExecutionEnvironmentTest.scala | 14 +-
.../api/scala/StreamingOperatorsITCase.scala | 9 +-
.../scala/StreamingScalaAPICompletenessTest.scala | 42 +-
.../api/scala/TimeWindowTranslationTest.scala | 21 +-
.../streaming/api/scala/WindowFunctionITCase.scala | 196 +-
.../streaming/api/scala/WindowReduceITCase.scala | 276 ++-
.../api/scala/WindowTranslationTest.scala | 358 ++-
.../scala/extensions/base/AcceptPFTestBase.scala | 16 +-
.../api/scala/extensions/data/KeyValuePair.scala | 14 +-
.../OnConnectedDataStreamTest.scala | 71 +-
.../acceptPartialFunctions/OnDataStreamTest.scala | 57 +-
.../OnJoinedStreamTest.scala | 45 +-
.../OnKeyedDataStreamTest.scala | 11 +-
.../OnWindowedStreamTest.scala | 11 +-
.../sink/filesystem/StreamingFileSinkTest.scala | 22 +-
.../CheckingIdentityRichAllWindowFunction.scala | 10 +-
...ckingIdentityRichProcessAllWindowFunction.scala | 2 -
...CheckingIdentityRichProcessWindowFunction.scala | 2 -
.../CheckingIdentityRichWindowFunction.scala | 13 +-
.../api/bridge/scala/DataStreamConversions.scala | 115 +-
.../api/bridge/scala/StreamStatementSet.scala | 4 +-
.../api/bridge/scala/StreamTableEnvironment.scala | 1118 ++++-----
.../table/api/bridge/scala/TableConversions.scala | 121 +-
.../scala/internal/StreamStatementSetImpl.scala | 16 +-
.../internal/StreamTableEnvironmentImpl.scala | 73 +-
.../flink/table/api/bridge/scala/package.scala | 14 +-
.../internal/StreamTableEnvironmentImplTest.scala | 29 +-
.../types/TypeInfoDataTypeConverterScalaTest.scala | 14 +-
.../table/api/ImplicitExpressionConversions.scala | 474 ++--
.../table/api/ImplicitExpressionOperations.scala | 169 +-
.../scala/org/apache/flink/table/api/package.scala | 41 +-
.../api/ExpressionsConsistencyCheckTest.scala | 51 +-
.../flink/table/api/ImplicitConversionsTest.scala | 15 +-
.../expressions/ObjectToExpressionScalaTest.scala | 5 +-
.../extraction/DataTypeExtractorScalaTest.scala | 52 +-
.../TypeInferenceExtractorScalaTest.scala | 66 +-
.../table/expressions/ExpressionParserImpl.scala | 341 +--
.../table/planner/calcite/CalciteConfig.scala | 92 +-
.../flink/table/planner/calcite/FlinkContext.scala | 9 +-
.../table/planner/calcite/FlinkContextImpl.scala | 1 -
.../table/planner/calcite/FlinkLocalRef.scala | 29 +-
.../planner/calcite/FlinkLogicalRelFactories.scala | 88 +-
.../table/planner/calcite/FlinkPlannerImpl.scala | 117 +-
.../table/planner/calcite/FlinkRelFactories.scala | 35 +-
.../calcite/FlinkRelOptClusterFactory.scala | 9 +-
.../table/planner/calcite/FlinkTypeFactory.scala | 238 +-
.../table/planner/calcite/FlinkTypeSystem.scala | 56 +-
.../planner/calcite/PreValidateReWriter.scala | 214 +-
.../table/planner/codegen/CalcCodeGenerator.scala | 22 +-
.../table/planner/codegen/CodeGenException.scala | 5 +-
.../flink/table/planner/codegen/CodeGenUtils.scala | 386 ++--
.../planner/codegen/CodeGeneratorContext.scala | 526 +++--
.../planner/codegen/CollectorCodeGenerator.scala | 70 +-
.../planner/codegen/CorrelateCodeGenerator.scala | 29 +-
.../planner/codegen/EqualiserCodeGenerator.scala | 28 +-
.../planner/codegen/ExpandCodeGenerator.scala | 28 +-
.../table/planner/codegen/ExprCodeGenerator.scala | 390 ++--
.../table/planner/codegen/ExpressionReducer.scala | 323 ++-
.../codegen/ExternalGeneratedExpression.scala | 51 +-
.../planner/codegen/FunctionCodeGenerator.scala | 221 +-
.../table/planner/codegen/GenerateUtils.scala | 401 ++--
.../planner/codegen/GeneratedExpression.scala | 69 +-
.../table/planner/codegen/HashCodeGenerator.scala | 76 +-
.../flink/table/planner/codegen/Indenter.scala | 10 +-
.../planner/codegen/InputFormatCodeGenerator.scala | 66 +-
.../table/planner/codegen/JsonGenerateUtils.scala | 75 +-
.../planner/codegen/LongHashJoinGenerator.scala | 99 +-
.../planner/codegen/LookupJoinCodeGenerator.scala | 215 +-
.../table/planner/codegen/MatchCodeGenerator.scala | 376 ++--
.../codegen/NestedLoopJoinCodeGenerator.scala | 119 +-
.../planner/codegen/OperatorCodeGenerator.scala | 91 +-
.../planner/codegen/ProjectionCodeGenerator.scala | 58 +-
.../table/planner/codegen/SinkCodeGenerator.scala | 38 +-
.../planner/codegen/ValuesCodeGenerator.scala | 18 +-
.../codegen/WatermarkGeneratorCodeGenerator.scala | 41 +-
.../table/planner/codegen/agg/AggCodeGen.scala | 16 +-
.../codegen/agg/AggsHandlerCodeGenerator.scala | 492 ++--
.../codegen/agg/DeclarativeAggCodeGen.scala | 131 +-
.../planner/codegen/agg/DistinctAggCodeGen.scala | 256 ++-
.../planner/codegen/agg/ImperativeAggCodeGen.scala | 145 +-
.../codegen/agg/batch/AggCodeGenHelper.scala | 617 +++--
.../agg/batch/AggWithoutKeysCodeGenerator.scala | 10 +-
.../codegen/agg/batch/HashAggCodeGenHelper.scala | 275 +--
.../codegen/agg/batch/HashAggCodeGenerator.scala | 67 +-
.../agg/batch/HashWindowCodeGenerator.scala | 246 +-
.../codegen/agg/batch/SortAggCodeGenerator.scala | 33 +-
.../agg/batch/SortWindowCodeGenerator.scala | 111 +-
.../codegen/agg/batch/WindowCodeGenerator.scala | 164 +-
.../codegen/calls/BridgingFunctionGenUtil.scala | 203 +-
.../codegen/calls/BridgingSqlFunctionCallGen.scala | 9 +-
.../planner/codegen/calls/BuiltInMethods.scala | 459 ++--
.../planner/codegen/calls/CallGenerator.scala | 13 +-
.../planner/codegen/calls/ConstantCallGen.scala | 7 +-
.../codegen/calls/CurrentTimePointCallGen.scala | 11 +-
.../table/planner/codegen/calls/DivCallGen.scala | 5 +-
.../planner/codegen/calls/ExtractCallGen.scala | 110 +-
.../planner/codegen/calls/FloorCeilCallGen.scala | 13 +-
.../planner/codegen/calls/FunctionGenerator.scala | 813 ++-----
.../planner/codegen/calls/HashCodeCallGen.scala | 5 +-
.../table/planner/codegen/calls/IfCallGen.scala | 54 +-
.../planner/codegen/calls/JsonArrayCallGen.scala | 51 +-
.../planner/codegen/calls/JsonObjectCallGen.scala | 75 +-
.../planner/codegen/calls/JsonStringCallGen.scala | 5 +-
.../planner/codegen/calls/JsonValueCallGen.scala | 89 +-
.../table/planner/codegen/calls/LikeCallGen.scala | 29 +-
.../planner/codegen/calls/MethodCallGen.scala | 59 +-
.../table/planner/codegen/calls/NotCallGen.scala | 11 +-
.../table/planner/codegen/calls/PrintCallGen.scala | 7 +-
.../table/planner/codegen/calls/RandCallGen.scala | 16 +-
.../codegen/calls/ScalarFunctionCallGen.scala | 85 +-
.../planner/codegen/calls/ScalarOperatorGens.scala | 856 ++++---
.../planner/codegen/calls/SearchOperatorGen.scala | 67 +-
.../planner/codegen/calls/StringCallGen.scala | 350 ++-
.../codegen/calls/TableFunctionCallGen.scala | 56 +-
.../codegen/calls/TimestampDiffCallGen.scala | 20 +-
...ltiFieldRangeBoundComparatorCodeGenerator.scala | 15 +-
.../over/RangeBoundComparatorCodeGenerator.scala | 56 +-
.../codegen/sort/ComparatorCodeGenerator.scala | 37 +-
.../planner/codegen/sort/SortCodeGenerator.scala | 99 +-
.../table/planner/delegation/BatchPlanner.scala | 34 +-
.../table/planner/delegation/PlannerBase.scala | 145 +-
.../table/planner/delegation/StreamPlanner.scala | 68 +-
.../planner/expressions/ExpressionBridge.scala | 5 +-
.../table/planner/expressions/InputTypeSpec.scala | 38 +-
.../planner/expressions/PlannerExpression.scala | 24 +-
.../expressions/PlannerExpressionConverter.scala | 68 +-
.../expressions/PlannerExpressionUtils.scala | 5 +-
.../table/planner/expressions/Reinterpret.scala | 16 +-
.../flink/table/planner/expressions/TreeNode.scala | 75 +-
.../table/planner/expressions/aggregations.scala | 53 +-
.../flink/table/planner/expressions/call.scala | 99 +-
.../table/planner/expressions/collection.scala | 13 +-
.../planner/expressions/fieldExpression.scala | 25 +-
.../flink/table/planner/expressions/literals.scala | 23 +-
.../table/planner/expressions/overOffsets.scala | 1 -
.../flink/table/planner/expressions/subquery.scala | 9 +-
.../flink/table/planner/expressions/symbols.scala | 27 +-
.../flink/table/planner/expressions/time.scala | 175 +-
.../planner/expressions/windowProperties.scala | 1 -
.../planner/functions/utils/AggSqlFunction.scala | 105 +-
.../functions/utils/ScalarSqlFunction.scala | 77 +-
.../planner/functions/utils/TableSqlFunction.scala | 144 +-
.../functions/utils/UserDefinedFunctionUtils.scala | 729 +++---
.../flink/table/planner/plan/cost/FlinkCost.scala | 272 ++-
.../table/planner/plan/cost/FlinkCostBase.scala | 13 +-
.../table/planner/plan/cost/FlinkCostFactory.scala | 5 +-
.../planner/plan/cost/FlinkCostFactoryBase.scala | 15 +-
.../planner/plan/logical/MatchRecognize.scala | 23 +-
.../table/planner/plan/logical/groupWindows.scala | 26 +-
.../metadata/AggCallSelectivityEstimator.scala | 177 +-
.../metadata/FlinkDefaultRelMetadataProvider.scala | 1 -
.../plan/metadata/FlinkRelMdColumnInterval.scala | 470 ++--
.../plan/metadata/FlinkRelMdColumnNullCount.scala | 170 +-
.../metadata/FlinkRelMdColumnOriginNullCount.scala | 20 +-
.../plan/metadata/FlinkRelMdColumnUniqueness.scala | 232 +-
.../plan/metadata/FlinkRelMdCumulativeCost.scala | 16 +-
.../plan/metadata/FlinkRelMdDistinctRowCount.scala | 156 +-
.../plan/metadata/FlinkRelMdDistribution.scala | 32 +-
.../FlinkRelMdFilteredColumnInterval.scala | 173 +-
.../metadata/FlinkRelMdModifiedMonotonicity.scala | 130 +-
.../metadata/FlinkRelMdNonCumulativeCost.scala | 10 +-
.../FlinkRelMdPercentageOriginalRows.scala | 29 +-
.../plan/metadata/FlinkRelMdPopulationSize.scala | 152 +-
.../planner/plan/metadata/FlinkRelMdRowCount.scala | 82 +-
.../plan/metadata/FlinkRelMdSelectivity.scala | 74 +-
.../planner/plan/metadata/FlinkRelMdSize.scala | 172 +-
.../plan/metadata/FlinkRelMdUniqueGroups.scala | 115 +-
.../plan/metadata/FlinkRelMdUniqueKeys.scala | 173 +-
.../plan/metadata/FlinkRelMdUpsertKeys.scala | 102 +-
.../plan/metadata/FlinkRelMdWindowProperties.scala | 122 +-
.../plan/metadata/SelectivityEstimator.scala | 582 ++---
.../planner/plan/nodes/FlinkConventions.scala | 13 +-
.../table/planner/plan/nodes/FlinkRelNode.scala | 36 +-
.../table/planner/plan/nodes/calcite/Expand.scala | 36 +-
.../planner/plan/nodes/calcite/LegacySink.scala | 27 +-
.../planner/plan/nodes/calcite/LogicalExpand.scala | 9 +-
.../plan/nodes/calcite/LogicalLegacySink.scala | 32 +-
.../planner/plan/nodes/calcite/LogicalRank.scala | 8 +-
.../planner/plan/nodes/calcite/LogicalSink.scala | 12 +-
.../plan/nodes/calcite/LogicalTableAggregate.scala | 11 +-
.../nodes/calcite/LogicalWatermarkAssigner.scala | 10 +-
.../nodes/calcite/LogicalWindowAggregate.scala | 3 +-
.../calcite/LogicalWindowTableAggregate.scala | 16 +-
.../table/planner/plan/nodes/calcite/Rank.scala | 71 +-
.../table/planner/plan/nodes/calcite/Sink.scala | 28 +-
.../plan/nodes/calcite/TableAggregate.scala | 25 +-
.../plan/nodes/calcite/WatermarkAssigner.scala | 35 +-
.../plan/nodes/calcite/WindowAggregate.scala | 45 +-
.../plan/nodes/calcite/WindowTableAggregate.scala | 23 +-
.../planner/plan/nodes/common/CommonCalc.scala | 44 +-
.../nodes/common/CommonIntermediateTableScan.scala | 9 +-
.../common/CommonPhysicalWindowTableFunction.scala | 10 +-
.../plan/nodes/logical/FlinkLogicalAggregate.scala | 37 +-
.../plan/nodes/logical/FlinkLogicalCalc.scala | 21 +-
.../plan/nodes/logical/FlinkLogicalCorrelate.scala | 15 +-
.../logical/FlinkLogicalDataStreamTableScan.scala | 34 +-
.../nodes/logical/FlinkLogicalDistribution.scala | 52 +-
.../plan/nodes/logical/FlinkLogicalExpand.scala | 19 +-
.../FlinkLogicalIntermediateTableScan.scala | 28 +-
.../plan/nodes/logical/FlinkLogicalIntersect.scala | 11 +-
.../plan/nodes/logical/FlinkLogicalJoin.scala | 22 +-
.../nodes/logical/FlinkLogicalLegacySink.scala | 25 +-
.../FlinkLogicalLegacyTableSourceScan.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalMatch.scala | 12 +-
.../plan/nodes/logical/FlinkLogicalMinus.scala | 7 +-
.../nodes/logical/FlinkLogicalOverAggregate.scala | 43 +-
.../plan/nodes/logical/FlinkLogicalRank.scala | 31 +-
.../plan/nodes/logical/FlinkLogicalRel.scala | 9 +-
.../plan/nodes/logical/FlinkLogicalSink.scala | 7 +-
.../plan/nodes/logical/FlinkLogicalSnapshot.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalSort.scala | 47 +-
.../nodes/logical/FlinkLogicalTableAggregate.scala | 16 +-
.../logical/FlinkLogicalTableFunctionScan.scala | 9 +-
.../logical/FlinkLogicalTableSourceScan.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalUnion.scala | 13 +-
.../plan/nodes/logical/FlinkLogicalValues.scala | 20 +-
.../logical/FlinkLogicalWatermarkAssigner.scala | 28 +-
.../logical/FlinkLogicalWindowAggregate.scala | 3 +-
.../logical/FlinkLogicalWindowTableAggregate.scala | 7 +-
.../plan/nodes/physical/FlinkPhysicalRel.scala | 24 +-
.../batch/BatchPhysicalBoundedStreamScan.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalCalc.scala | 7 +-
.../physical/batch/BatchPhysicalCalcBase.scala | 13 +-
.../physical/batch/BatchPhysicalCorrelate.scala | 22 +-
.../batch/BatchPhysicalCorrelateBase.scala | 42 +-
.../physical/batch/BatchPhysicalExchange.scala | 16 +-
.../nodes/physical/batch/BatchPhysicalExpand.scala | 7 +-
.../batch/BatchPhysicalGroupAggregateBase.scala | 21 +-
.../batch/BatchPhysicalHashAggregate.scala | 44 +-
.../batch/BatchPhysicalHashAggregateBase.scala | 11 +-
.../physical/batch/BatchPhysicalHashJoin.scala | 21 +-
.../batch/BatchPhysicalHashWindowAggregate.scala | 12 +-
.../BatchPhysicalHashWindowAggregateBase.scala | 7 +-
.../batch/BatchPhysicalIntermediateTableScan.scala | 7 +-
.../physical/batch/BatchPhysicalJoinBase.scala | 134 +-
.../physical/batch/BatchPhysicalLegacySink.scala | 6 +-
.../batch/BatchPhysicalLegacyTableSourceScan.scala | 7 +-
.../nodes/physical/batch/BatchPhysicalLimit.scala | 11 +-
.../batch/BatchPhysicalLocalHashAggregate.scala | 35 +-
.../BatchPhysicalLocalHashWindowAggregate.scala | 9 +-
.../batch/BatchPhysicalLocalSortAggregate.scala | 38 +-
.../BatchPhysicalLocalSortWindowAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalLookupJoin.scala | 7 +-
.../batch/BatchPhysicalNestedLoopJoin.scala | 21 +-
.../batch/BatchPhysicalOverAggregate.scala | 7 +-
.../batch/BatchPhysicalOverAggregateBase.scala | 72 +-
.../physical/batch/BatchPhysicalPythonCalc.scala | 16 +-
.../batch/BatchPhysicalPythonCorrelate.scala | 18 +-
.../batch/BatchPhysicalPythonGroupAggregate.scala | 44 +-
.../BatchPhysicalPythonGroupWindowAggregate.scala | 9 +-
.../batch/BatchPhysicalPythonOverAggregate.scala | 3 +-
.../nodes/physical/batch/BatchPhysicalRank.scala | 16 +-
.../nodes/physical/batch/BatchPhysicalRel.scala | 10 +-
.../nodes/physical/batch/BatchPhysicalSink.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalSort.scala | 13 +-
.../batch/BatchPhysicalSortAggregate.scala | 44 +-
.../batch/BatchPhysicalSortAggregateBase.scala | 5 +-
.../physical/batch/BatchPhysicalSortLimit.scala | 21 +-
.../batch/BatchPhysicalSortMergeJoin.scala | 58 +-
.../batch/BatchPhysicalSortWindowAggregate.scala | 5 +-
.../BatchPhysicalSortWindowAggregateBase.scala | 3 +-
.../batch/BatchPhysicalTableSourceScan.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalUnion.scala | 21 +-
.../nodes/physical/batch/BatchPhysicalValues.scala | 10 +-
.../batch/BatchPhysicalWindowAggregateBase.scala | 41 +-
.../batch/BatchPhysicalWindowTableFunction.scala | 23 +-
.../physical/common/CommonPhysicalExchange.scala | 20 +-
.../nodes/physical/common/CommonPhysicalJoin.scala | 43 +-
.../CommonPhysicalLegacyTableSourceScan.scala | 10 +-
.../physical/common/CommonPhysicalLookupJoin.scala | 193 +-
.../common/CommonPhysicalTableSourceScan.scala | 11 +-
.../nodes/physical/stream/StreamPhysicalCalc.scala | 9 +-
.../physical/stream/StreamPhysicalCalcBase.scala | 7 +-
.../stream/StreamPhysicalChangelogNormalize.scala | 10 +-
.../physical/stream/StreamPhysicalCorrelate.scala | 22 +-
.../stream/StreamPhysicalCorrelateBase.scala | 30 +-
.../stream/StreamPhysicalDataStreamScan.scala | 13 +-
.../stream/StreamPhysicalDeduplicate.scala | 12 +-
.../stream/StreamPhysicalDropUpdateBefore.scala | 24 +-
.../physical/stream/StreamPhysicalExchange.scala | 7 +-
.../physical/stream/StreamPhysicalExpand.scala | 9 +-
.../StreamPhysicalGlobalGroupAggregate.scala | 36 +-
.../StreamPhysicalGlobalWindowAggregate.scala | 52 +-
.../stream/StreamPhysicalGroupAggregate.scala | 22 +-
.../stream/StreamPhysicalGroupAggregateBase.scala | 45 +-
.../stream/StreamPhysicalGroupTableAggregate.scala | 8 +-
.../StreamPhysicalGroupTableAggregateBase.scala | 27 +-
.../StreamPhysicalGroupWindowAggregate.scala | 9 +-
.../StreamPhysicalGroupWindowAggregateBase.scala | 39 +-
.../StreamPhysicalGroupWindowTableAggregate.scala | 5 +-
.../StreamPhysicalIncrementalGroupAggregate.scala | 36 +-
.../StreamPhysicalIntermediateTableScan.scala | 7 +-
.../stream/StreamPhysicalIntervalJoin.scala | 63 +-
.../nodes/physical/stream/StreamPhysicalJoin.scala | 59 +-
.../physical/stream/StreamPhysicalLegacySink.scala | 6 +-
.../StreamPhysicalLegacyTableSourceScan.scala | 5 +-
.../physical/stream/StreamPhysicalLimit.scala | 18 +-
.../stream/StreamPhysicalLocalGroupAggregate.scala | 27 +-
.../StreamPhysicalLocalWindowAggregate.scala | 50 +-
.../physical/stream/StreamPhysicalLookupJoin.scala | 6 +-
.../physical/stream/StreamPhysicalMatch.scala | 45 +-
.../stream/StreamPhysicalMiniBatchAssigner.scala | 11 +-
.../stream/StreamPhysicalOverAggregate.scala | 15 +-
.../stream/StreamPhysicalOverAggregateBase.scala | 32 +-
.../physical/stream/StreamPhysicalPythonCalc.scala | 16 +-
.../stream/StreamPhysicalPythonCorrelate.scala | 18 +-
.../StreamPhysicalPythonGroupAggregate.scala | 29 +-
.../StreamPhysicalPythonGroupTableAggregate.scala | 8 +-
.../StreamPhysicalPythonGroupWindowAggregate.scala | 9 +-
.../stream/StreamPhysicalPythonOverAggregate.scala | 15 +-
.../nodes/physical/stream/StreamPhysicalRank.scala | 7 +-
.../nodes/physical/stream/StreamPhysicalRel.scala | 9 +-
.../nodes/physical/stream/StreamPhysicalSink.scala | 19 +-
.../nodes/physical/stream/StreamPhysicalSort.scala | 25 +-
.../physical/stream/StreamPhysicalSortLimit.scala | 20 +-
.../stream/StreamPhysicalTableSourceScan.scala | 1 -
.../stream/StreamPhysicalTemporalJoin.scala | 67 +-
.../stream/StreamPhysicalTemporalSort.scala | 15 +-
.../physical/stream/StreamPhysicalUnion.scala | 9 +-
.../physical/stream/StreamPhysicalValues.scala | 7 +-
.../stream/StreamPhysicalWatermarkAssigner.scala | 25 +-
.../stream/StreamPhysicalWindowAggregate.scala | 32 +-
.../stream/StreamPhysicalWindowDeduplicate.scala | 11 +-
.../physical/stream/StreamPhysicalWindowJoin.scala | 36 +-
.../physical/stream/StreamPhysicalWindowRank.scala | 6 +-
.../stream/StreamPhysicalWindowTableFunction.scala | 16 +-
.../BatchCommonSubGraphBasedOptimizer.scala | 59 +-
.../optimize/CommonSubGraphBasedOptimizer.scala | 105 +-
.../table/planner/plan/optimize/Optimizer.scala | 21 +-
.../table/planner/plan/optimize/RelNodeBlock.scala | 249 +-
.../StreamCommonSubGraphBasedOptimizer.scala | 144 +-
.../optimize/program/BatchOptimizeContext.scala | 9 +-
.../plan/optimize/program/FlinkBatchProgram.scala | 209 +-
.../optimize/program/FlinkChainedProgram.scala | 87 +-
.../FlinkChangelogModeInferenceProgram.scala | 716 +++---
.../optimize/program/FlinkDecorrelateProgram.scala | 29 +-
.../plan/optimize/program/FlinkGroupProgram.scala | 30 +-
.../plan/optimize/program/FlinkHepProgram.scala | 37 +-
.../optimize/program/FlinkHepRuleSetProgram.scala | 109 +-
.../FlinkMiniBatchIntervalTraitInitProgram.scala | 9 +-
.../optimize/program/FlinkOptimizeContext.scala | 15 +-
.../optimize/program/FlinkOptimizeProgram.scala | 16 +-
.../program/FlinkRelTimeIndicatorProgram.scala | 10 +-
.../optimize/program/FlinkRuleSetProgram.scala | 36 +-
.../plan/optimize/program/FlinkStreamProgram.scala | 219 +-
.../optimize/program/FlinkVolcanoProgram.scala | 39 +-
.../optimize/program/StreamOptimizeContext.scala | 13 +-
.../table/planner/plan/reuse/SubplanReuser.scala | 146 +-
.../planner/plan/rules/FlinkBatchRuleSets.scala | 150 +-
.../planner/plan/rules/FlinkStreamRuleSets.scala | 206 +-
.../rules/common/CommonTemporalTableJoinRule.scala | 29 +-
.../logical/AggregateReduceGroupingRule.scala | 48 +-
.../logical/BatchLogicalWindowAggregateRule.scala | 20 +-
.../plan/rules/logical/CalcRankTransposeRule.scala | 44 +-
.../rules/logical/CalcSnapshotTransposeRule.scala | 14 +-
.../ConstantRankNumberColumnRemoveRule.scala | 9 +-
.../rules/logical/ConvertToNotInOrInRule.scala | 91 +-
.../rules/logical/CorrelateSortToRankRule.scala | 58 +-
.../rules/logical/DecomposeGroupingSetsRule.scala | 333 ++-
.../logical/EnumerableToLogicalTableScan.scala | 19 +-
.../plan/rules/logical/FlinkCalcMergeRule.scala | 63 +-
.../plan/rules/logical/FlinkLimit0RemoveRule.scala | 12 +-
.../plan/rules/logical/FlinkLogicalRankRule.scala | 77 +-
.../plan/rules/logical/FlinkPruneEmptyRules.scala | 33 +-
.../rules/logical/FlinkRewriteSubQueryRule.scala | 69 +-
.../rules/logical/FlinkSubQueryRemoveRule.scala | 85 +-
.../JoinConditionEqualityTransferRule.scala | 91 +-
.../logical/JoinConditionTypeCoerceRule.scala | 52 +-
.../JoinDependentConditionDerivationRule.scala | 127 +-
.../rules/logical/JoinDeriveNullFilterRule.scala | 49 +-
...relateToJoinFromTemporalTableFunctionRule.scala | 102 +-
...gicalCorrelateToJoinFromTemporalTableRule.scala | 150 +-
.../plan/rules/logical/LogicalUnnestRule.scala | 36 +-
.../logical/LogicalWindowAggregateRuleBase.scala | 196 +-
.../logical/ProjectSemiAntiJoinTransposeRule.scala | 46 +-
.../rules/logical/PruneAggregateCallRule.scala | 78 +-
.../PushFilterIntoLegacyTableSourceScanRule.scala | 42 +-
.../PushLimitIntoLegacyTableSourceScanRule.scala | 61 +-
...ushPartitionIntoLegacyTableSourceScanRule.scala | 85 +-
.../PushProjectIntoLegacyTableSourceScanRule.scala | 69 +-
.../plan/rules/logical/PythonCalcSplitRule.scala | 288 +--
.../logical/ReplaceIntersectWithSemiJoinRule.scala | 22 +-
.../logical/ReplaceMinusWithAntiJoinRule.scala | 22 +-
.../rules/logical/RewriteIntersectAllRule.scala | 147 +-
.../plan/rules/logical/RewriteMinusAllRule.scala | 107 +-
.../logical/RewriteMultiJoinConditionRule.scala | 53 +-
.../logical/SimplifyFilterConditionRule.scala | 36 +-
.../rules/logical/SimplifyJoinConditionRule.scala | 17 +-
.../plan/rules/logical/SplitAggregateRule.scala | 465 ++--
.../SplitPythonConditionFromCorrelateRule.scala | 45 +-
.../logical/SplitPythonConditionFromJoinRule.scala | 27 +-
.../logical/StreamLogicalWindowAggregateRule.scala | 27 +-
.../TemporalJoinRewriteWithUniqueKeyRule.scala | 79 +-
.../WindowAggregateReduceFunctionsRule.scala | 19 +-
.../rules/logical/WindowGroupReorderRule.scala | 57 +-
.../plan/rules/logical/WindowPropertiesRule.scala | 103 +-
.../rules/physical/FlinkExpandConversionRule.scala | 21 +-
.../physical/batch/BatchPhysicalAggRuleBase.scala | 51 +-
.../batch/BatchPhysicalBoundedStreamScanRule.scala | 9 +-
.../physical/batch/BatchPhysicalCalcRule.scala | 15 +-
...atchPhysicalConstantTableFunctionScanRule.scala | 32 +-
.../batch/BatchPhysicalCorrelateRule.scala | 14 +-
.../batch/BatchPhysicalDistributionRule.scala | 31 +-
.../physical/batch/BatchPhysicalExpandRule.scala | 5 +-
.../physical/batch/BatchPhysicalHashAggRule.scala | 120 +-
.../physical/batch/BatchPhysicalHashJoinRule.scala | 64 +-
.../BatchPhysicalIntermediateTableScanRule.scala | 1 -
.../physical/batch/BatchPhysicalJoinRuleBase.scala | 38 +-
.../batch/BatchPhysicalLegacySinkRule.scala | 25 +-
.../BatchPhysicalLegacyTableSourceScanRule.scala | 1 -
.../physical/batch/BatchPhysicalLimitRule.scala | 34 +-
.../batch/BatchPhysicalLookupJoinRule.scala | 16 +-
.../batch/BatchPhysicalNestedLoopJoinRule.scala | 11 +-
.../batch/BatchPhysicalOverAggregateRule.scala | 63 +-
.../batch/BatchPhysicalPythonCalcRule.scala | 19 +-
.../physical/batch/BatchPhysicalRankRule.scala | 24 +-
.../batch/BatchPhysicalSingleRowJoinRule.scala | 22 +-
.../physical/batch/BatchPhysicalSinkRule.scala | 35 +-
.../physical/batch/BatchPhysicalSortAggRule.scala | 104 +-
.../batch/BatchPhysicalSortLimitRule.scala | 36 +-
.../batch/BatchPhysicalSortMergeJoinRule.scala | 73 +-
.../physical/batch/BatchPhysicalSortRule.scala | 31 +-
.../batch/BatchPhysicalTableSourceScanRule.scala | 5 +-
.../physical/batch/BatchPhysicalUnionRule.scala | 12 +-
.../physical/batch/BatchPhysicalValuesRule.scala | 22 +-
.../batch/BatchPhysicalWindowAggregateRule.scala | 108 +-
.../BatchPhysicalWindowTableFunctionRule.scala | 16 +-
.../physical/batch/EnforceLocalAggRuleBase.scala | 27 +-
.../physical/batch/EnforceLocalHashAggRule.scala | 15 +-
.../physical/batch/EnforceLocalSortAggRule.scala | 73 +-
.../batch/RemoveRedundantLocalHashAggRule.scala | 16 +-
.../batch/RemoveRedundantLocalRankRule.scala | 30 +-
.../batch/RemoveRedundantLocalSortAggRule.scala | 46 +-
.../physical/common/CommonLookupJoinRule.scala | 40 +-
.../ExpandWindowTableFunctionTransposeRule.scala | 101 +-
.../physical/stream/IncrementalAggregateRule.scala | 55 +-
.../stream/MiniBatchIntervalInferRule.scala | 115 +-
...indowTableFunctionIntoWindowAggregateRule.scala | 46 +-
.../physical/stream/StreamPhysicalCalcRule.scala | 12 +-
...reamPhysicalConstantTableFunctionScanRule.scala | 32 +-
.../stream/StreamPhysicalCorrelateRule.scala | 13 +-
.../stream/StreamPhysicalDataStreamScanRule.scala | 5 +-
.../stream/StreamPhysicalDeduplicateRule.scala | 54 +-
.../physical/stream/StreamPhysicalExpandRule.scala | 5 +-
.../stream/StreamPhysicalGroupAggregateRule.scala | 8 +-
.../StreamPhysicalGroupTableAggregateRule.scala | 7 +-
.../StreamPhysicalGroupWindowAggregateRule.scala | 5 +-
...reamPhysicalGroupWindowTableAggregateRule.scala | 7 +-
.../StreamPhysicalIntermediateTableScanRule.scala | 1 -
.../stream/StreamPhysicalIntervalJoinRule.scala | 40 +-
.../physical/stream/StreamPhysicalJoinRule.scala | 14 +-
.../stream/StreamPhysicalJoinRuleBase.scala | 17 +-
.../stream/StreamPhysicalLegacySinkRule.scala | 23 +-
.../StreamPhysicalLegacyTableSourceScanRule.scala | 1 -
.../physical/stream/StreamPhysicalLimitRule.scala | 9 +-
.../stream/StreamPhysicalLookupJoinRule.scala | 24 +-
.../physical/stream/StreamPhysicalMatchRule.scala | 28 +-
.../stream/StreamPhysicalOverAggregateRule.scala | 7 +-
.../stream/StreamPhysicalPythonCalcRule.scala | 12 +-
.../physical/stream/StreamPhysicalRankRule.scala | 7 +-
.../physical/stream/StreamPhysicalSinkRule.scala | 36 +-
.../stream/StreamPhysicalSortLimitRule.scala | 9 +-
.../physical/stream/StreamPhysicalSortRule.scala | 14 +-
.../stream/StreamPhysicalTableSourceScanRule.scala | 22 +-
.../stream/StreamPhysicalTemporalJoinRule.scala | 5 +-
.../stream/StreamPhysicalTemporalSortRule.scala | 34 +-
.../physical/stream/StreamPhysicalUnionRule.scala | 12 +-
.../physical/stream/StreamPhysicalValuesRule.scala | 11 +-
.../StreamPhysicalWatermarkAssignerRule.scala | 15 +-
.../stream/StreamPhysicalWindowAggregateRule.scala | 69 +-
.../StreamPhysicalWindowDeduplicateRule.scala | 14 +-
.../stream/StreamPhysicalWindowJoinRule.scala | 18 +-
.../stream/StreamPhysicalWindowRankRule.scala | 12 +-
.../StreamPhysicalWindowTableFunctionRule.scala | 16 +-
.../stream/TwoStageOptimizedAggregateRule.scala | 71 +-
.../planner/plan/schema/DataStreamTable.scala | 37 +-
.../schema/DeferredTypeFlinkTableFunction.scala | 20 +-
.../planner/plan/schema/FlinkTableFunction.scala | 21 +-
.../planner/plan/schema/GenericRelDataType.scala | 35 +-
.../planner/plan/schema/IntermediateRelTable.scala | 25 +-
.../plan/schema/LegacyCatalogSourceTable.scala | 126 +-
.../plan/schema/LegacyTableSourceTable.scala | 90 +-
.../planner/plan/schema/TableSourceTable.scala | 66 +-
.../plan/schema/TimeIndicatorRelDataType.scala | 15 +-
.../plan/schema/TypedFlinkTableFunction.scala | 26 +-
.../table/planner/plan/stats/FlinkStatistic.scala | 102 +-
.../table/planner/plan/stats/ValueInterval.scala | 158 +-
.../planner/plan/trait/FlinkRelDistribution.scala | 96 +-
.../plan/trait/MiniBatchIntervalTrait.scala | 13 +-
.../plan/trait/MiniBatchIntervalTraitDef.scala | 1 -
.../planner/plan/trait/ModifyKindSetTrait.scala | 25 +-
.../planner/plan/trait/ModifyKindSetTraitDef.scala | 3 -
.../plan/trait/RelModifiedMonotonicity.scala | 8 +-
.../flink/table/planner/plan/trait/TraitUtil.scala | 42 +-
.../table/planner/plan/trait/UpdateKindTrait.scala | 34 +-
.../planner/plan/trait/UpdateKindTraitDef.scala | 5 +-
.../planner/plan/utils/AggFunctionFactory.scala | 120 +-
.../table/planner/plan/utils/AggregateUtil.scala | 783 +++----
.../planner/plan/utils/ChangelogPlanUtils.scala | 37 +-
.../planner/plan/utils/ColumnIntervalUtil.scala | 123 +-
.../table/planner/plan/utils/CorrelateUtil.scala | 69 +-
.../table/planner/plan/utils/ExpandUtil.scala | 170 +-
.../table/planner/plan/utils/FlinkRelMdUtil.scala | 421 ++--
.../table/planner/plan/utils/FlinkRelOptUtil.scala | 371 +--
.../table/planner/plan/utils/FlinkRexUtil.scala | 255 +--
.../planner/plan/utils/IntervalJoinUtil.scala | 223 +-
.../flink/table/planner/plan/utils/JoinUtil.scala | 109 +-
.../flink/table/planner/plan/utils/MatchUtil.scala | 33 +-
.../planner/plan/utils/NestedProjectionUtil.scala | 177 +-
.../planner/plan/utils/OverAggregateUtil.scala | 119 +-
.../table/planner/plan/utils/PartitionPruner.scala | 78 +-
.../table/planner/plan/utils/PythonUtil.scala | 116 +-
.../flink/table/planner/plan/utils/RankUtil.scala | 146 +-
.../table/planner/plan/utils/RelExplainUtil.scala | 813 +++----
.../table/planner/plan/utils/RelShuttles.scala | 71 +-
.../planner/plan/utils/RelTreeWriterImpl.scala | 29 +-
.../planner/plan/utils/RexDefaultVisitor.scala | 7 +-
.../planner/plan/utils/RexNodeExtractor.scala | 254 ++-
.../table/planner/plan/utils/RexNodeRewriter.scala | 32 +-
.../flink/table/planner/plan/utils/ScanUtil.scala | 65 +-
.../planner/plan/utils/SetOpRewriteUtil.scala | 46 +-
.../flink/table/planner/plan/utils/SortUtil.scala | 67 +-
.../planner/plan/utils/TemporalJoinUtil.scala | 150 +-
.../planner/plan/utils/UpdatingPlanChecker.scala | 11 +-
.../planner/plan/utils/WindowEmitStrategy.scala | 65 +-
.../table/planner/plan/utils/WindowJoinUtil.scala | 185 +-
.../table/planner/plan/utils/WindowUtil.scala | 161 +-
.../table/planner/plan/utils/aggregation.scala | 102 +-
.../table/planner/sinks/CollectTableSink.scala | 14 +-
.../table/planner/sinks/DataStreamTableSink.scala | 35 +-
.../flink/table/planner/sinks/TableSinkUtils.scala | 210 +-
.../table/planner/sources/TableSourceUtil.scala | 199 +-
.../planner/typeutils/LegacyDataViewUtils.scala | 54 +-
.../table/planner/typeutils/TypeCoercion.scala | 199 +-
.../planner/typeutils/TypeInfoCheckUtils.scala | 74 +-
.../planner/utils/JavaScalaConversionUtil.scala | 7 +-
.../apache/flink/table/planner/utils/Logging.scala | 5 +-
.../table/planner/validate/ValidationResult.scala | 19 +-
.../apache/flink/table/api/ExecuteSqlTest.scala | 46 +-
.../flink/table/api/TableEnvironmentITCase.scala | 120 +-
.../flink/table/api/TableEnvironmentTest.scala | 423 ++--
.../org/apache/flink/table/api/TableITCase.scala | 73 +-
.../apache/flink/table/api/batch/ExplainTest.scala | 11 +-
.../flink/table/api/stream/ExplainTest.scala | 80 +-
.../validation/MatchRecognizeValidationTest.scala | 40 +-
.../sql/validation/OverWindowValidationTest.scala | 9 +-
.../planner/calcite/CalciteConfigBuilderTest.scala | 22 +-
.../planner/calcite/FlinkTypeFactoryTest.scala | 31 +-
.../table/planner/catalog/CatalogTableITCase.scala | 251 ++-
.../table/planner/catalog/CatalogTableTest.scala | 11 +-
.../table/planner/catalog/CatalogViewITCase.scala | 197 +-
.../planner/codegen/HashCodeGeneratorTest.scala | 35 +-
.../codegen/ProjectionCodeGeneratorTest.scala | 102 +-
.../codegen/WatermarkGeneratorCodeGenTest.scala | 51 +-
.../table/planner/codegen/agg/AggTestBase.scala | 20 +-
.../codegen/agg/AggsHandlerCodeGeneratorTest.scala | 37 +-
.../codegen/agg/batch/AggWithoutKeysTest.scala | 73 +-
.../codegen/agg/batch/BatchAggTestBase.scala | 46 +-
.../agg/batch/HashAggCodeGeneratorTest.scala | 89 +-
.../agg/batch/SortAggCodeGeneratorTest.scala | 96 +-
.../table/planner/expressions/ArrayTypeTest.scala | 248 +-
.../planner/expressions/CompositeAccessTest.scala | 36 +-
.../planner/expressions/DecimalTypeTest.scala | 641 ++----
.../planner/expressions/KeywordParseTest.scala | 6 +-
.../table/planner/expressions/LiteralTest.scala | 87 +-
.../table/planner/expressions/MapTypeTest.scala | 207 +-
.../expressions/NonDeterministicTests.scala | 70 +-
.../table/planner/expressions/RowTypeTest.scala | 21 +-
.../planner/expressions/ScalarFunctionsTest.scala | 2381 ++++++--------------
.../planner/expressions/ScalarOperatorsTest.scala | 76 +-
.../planner/expressions/SqlExpressionTest.scala | 113 +-
.../planner/expressions/TemporalTypesTest.scala | 638 ++----
.../expressions/utils/ArrayTypeTestBase.scala | 21 +-
.../expressions/utils/CompositeTypeTestBase.scala | 22 +-
.../expressions/utils/ExpressionTestBase.scala | 82 +-
.../expressions/utils/MapTypeTestBase.scala | 21 +-
.../expressions/utils/RowTypeTestBase.scala | 7 +-
.../utils/ScalarOperatorsTestBase.scala | 60 +-
.../expressions/utils/ScalarTypesTestBase.scala | 21 +-
.../utils/userDefinedScalarFunctions.scala | 6 +-
.../validation/ArrayTypeValidationTest.scala | 10 +-
.../validation/CompositeAccessValidationTest.scala | 3 -
.../validation/MapTypeValidationTest.scala | 1 -
.../validation/RowTypeValidationTest.scala | 1 -
.../validation/ScalarFunctionsValidationTest.scala | 50 +-
.../validation/ScalarOperatorsValidationTest.scala | 19 +-
.../utils/TestCollectionTableFactory.scala | 77 +-
.../planner/match/PatternTranslatorTest.scala | 94 +-
.../planner/match/PatternTranslatorTestBase.scala | 38 +-
.../table/planner/plan/batch/sql/CalcTest.scala | 6 +-
.../plan/batch/sql/CompactManagedTableTest.scala | 5 +-
.../plan/batch/sql/DagOptimizationTest.scala | 39 +-
.../plan/batch/sql/DeadlockBreakupTest.scala | 128 +-
.../planner/plan/batch/sql/LegacyLimitTest.scala | 2 +-
.../planner/plan/batch/sql/LegacySinkTest.scala | 41 +-
.../plan/batch/sql/LegacyTableSourceTest.scala | 82 +-
.../table/planner/plan/batch/sql/LimitTest.scala | 31 +-
.../plan/batch/sql/MultipleInputCreationTest.scala | 93 +-
.../plan/batch/sql/PartitionableSinkTest.scala | 1 -
.../plan/batch/sql/PartitionableSourceTest.scala | 46 +-
.../table/planner/plan/batch/sql/RankTest.scala | 44 +-
.../plan/batch/sql/RemoveCollationTest.scala | 110 +-
.../planner/plan/batch/sql/RemoveShuffleTest.scala | 217 +-
.../planner/plan/batch/sql/SetOperatorsTest.scala | 11 +-
.../planner/plan/batch/sql/SortLimitTest.scala | 5 +-
.../table/planner/plan/batch/sql/SortTest.scala | 27 +-
.../planner/plan/batch/sql/SubplanReuseTest.scala | 144 +-
.../planner/plan/batch/sql/TableScanTest.scala | 200 +-
.../planner/plan/batch/sql/TableSinkTest.scala | 145 +-
.../planner/plan/batch/sql/TableSourceTest.scala | 132 +-
.../table/planner/plan/batch/sql/UnionTest.scala | 1 -
.../table/planner/plan/batch/sql/ValuesTest.scala | 1 -
.../plan/batch/sql/WindowTableFunctionTest.scala | 33 +-
.../sql/agg/AggregateReduceGroupingTest.scala | 4 +-
.../plan/batch/sql/agg/AggregateTestBase.scala | 169 +-
.../plan/batch/sql/agg/DistinctAggregateTest.scala | 5 +-
.../plan/batch/sql/agg/GroupWindowTest.scala | 49 +-
.../plan/batch/sql/agg/GroupingSetsTest.scala | 22 +-
.../plan/batch/sql/agg/HashAggregateTest.scala | 8 +-
.../plan/batch/sql/agg/OverAggregateTest.scala | 10 +-
.../plan/batch/sql/agg/SortAggregateTest.scala | 8 +-
.../batch/sql/join/BroadcastHashJoinTest.scala | 5 +-
.../sql/join/BroadcastHashSemiAntiJoinTest.scala | 5 +-
.../planner/plan/batch/sql/join/JoinTestBase.scala | 2 +-
.../plan/batch/sql/join/LookupJoinTest.scala | 87 +-
.../plan/batch/sql/join/NestedLoopJoinTest.scala | 5 +-
.../sql/join/NestedLoopSemiAntiJoinTest.scala | 5 +-
.../plan/batch/sql/join/SemiAntiJoinTest.scala | 17 +-
.../plan/batch/sql/join/SemiAntiJoinTestBase.scala | 3 +-
.../plan/batch/sql/join/ShuffledHashJoinTest.scala | 1 -
.../sql/join/ShuffledHashSemiAntiJoinTest.scala | 1 -
.../plan/batch/sql/join/SingleRowJoinTest.scala | 1 -
.../plan/batch/sql/join/SortMergeJoinTest.scala | 5 +-
.../batch/sql/join/SortMergeSemiAntiJoinTest.scala | 5 +-
.../batch/sql/join/TemporalFunctionJoinTest.scala | 30 +-
.../plan/batch/sql/join/TemporalJoinTest.scala | 94 +-
.../planner/plan/batch/table/AggregateTest.scala | 22 +-
.../table/planner/plan/batch/table/CalcTest.scala | 12 +-
.../plan/batch/table/ColumnFunctionsTest.scala | 8 +-
.../planner/plan/batch/table/CorrelateTest.scala | 32 +-
.../planner/plan/batch/table/GroupWindowTest.scala | 27 +-
.../table/planner/plan/batch/table/JoinTest.scala | 39 +-
.../plan/batch/table/PythonAggregateTest.scala | 7 +-
.../planner/plan/batch/table/PythonCalcTest.scala | 1 -
.../table/PythonGroupWindowAggregateTest.scala | 45 +-
.../table/PythonOverWindowAggregateTest.scala | 44 +-
.../plan/batch/table/SetOperatorsTest.scala | 34 +-
.../table/TemporalTableFunctionJoinTest.scala | 14 +-
.../stringexpr/AggregateStringExpressionTest.scala | 101 +-
.../stringexpr/CalcStringExpressionTest.scala | 190 +-
.../stringexpr/CorrelateStringExpressionTest.scala | 20 +-
.../stringexpr/JoinStringExpressionTest.scala | 39 +-
.../batch/table/stringexpr/SetOperatorsTest.scala | 1 -
.../stringexpr/SortStringExpressionTest.scala | 3 +-
.../table/validation/AggregateValidationTest.scala | 35 +-
.../table/validation/CalcValidationTest.scala | 24 +-
.../table/validation/CorrelateValidationTest.scala | 12 +-
.../validation/GroupWindowValidationTest.scala | 31 +-
.../table/validation/JoinValidationTest.scala | 26 +-
.../validation/OverWindowValidationTest.scala | 15 +-
.../validation/SetOperatorsValidationTest.scala | 18 +-
.../table/validation/SortValidationTest.scala | 1 -
.../common/AggregateReduceGroupingTestBase.scala | 158 +-
.../plan/common/DistinctAggregateTestBase.scala | 16 +-
.../planner/plan/common/JoinReorderTestBase.scala | 188 +-
.../planner/plan/common/PartialInsertTest.scala | 74 +-
.../planner/plan/common/TableFactoryTest.scala | 11 +-
.../table/planner/plan/common/UnnestTestBase.scala | 12 +-
.../planner/plan/common/ViewsExpandingTest.scala | 99 +-
.../table/planner/plan/cost/FlinkCostTest.scala | 36 +-
.../table/planner/plan/hint/OptionsHintTest.scala | 79 +-
.../metadata/AggCallSelectivityEstimatorTest.scala | 373 ++-
.../metadata/FlinkRelMdColumnIntervalTest.scala | 365 +--
.../metadata/FlinkRelMdColumnNullCountTest.scala | 147 +-
.../FlinkRelMdColumnOriginNullCountTest.scala | 68 +-
.../metadata/FlinkRelMdColumnUniquenessTest.scala | 467 ++--
.../metadata/FlinkRelMdCumulativeCostTest.scala | 29 +-
.../metadata/FlinkRelMdDistinctRowCountTest.scala | 681 +++---
.../plan/metadata/FlinkRelMdDistributionTest.scala | 21 +-
.../FlinkRelMdFilteredColumnIntervalTest.scala | 253 ++-
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 1223 ++++++----
.../FlinkRelMdModifiedMonotonicityTest.scala | 160 +-
.../metadata/FlinkRelMdNonCumulativeCostTest.scala | 27 +-
.../FlinkRelMdPercentageOriginalRowsTest.scala | 44 +-
.../metadata/FlinkRelMdPopulationSizeTest.scala | 325 +--
.../plan/metadata/FlinkRelMdRowCollationTest.scala | 94 +-
.../plan/metadata/FlinkRelMdRowCountTest.scala | 197 +-
.../plan/metadata/FlinkRelMdSelectivityTest.scala | 536 +++--
.../planner/plan/metadata/FlinkRelMdSizeTest.scala | 181 +-
.../plan/metadata/FlinkRelMdUniqueGroupsTest.scala | 684 ++++--
.../plan/metadata/FlinkRelMdUniqueKeysTest.scala | 166 +-
.../plan/metadata/FlinkRelMdUpsertKeysTest.scala | 181 +-
.../metadata/MetadataHandlerConsistencyTest.scala | 95 +-
.../planner/plan/metadata/MetadataTestUtil.scala | 131 +-
.../plan/metadata/SelectivityEstimatorTest.scala | 794 ++++---
.../planner/plan/nodes/calcite/ExpandTest.scala | 9 +-
.../plan/nodes/calcite/RelNodeTestBase.scala | 20 +-
.../optimize/program/FlinkChainedProgramTest.scala | 36 +-
.../program/FlinkHepRuleSetProgramTest.scala | 44 +-
.../optimize/program/FlinkVolcanoProgramTest.scala | 18 +-
.../logical/AggregateReduceGroupingRuleTest.scala | 9 +-
.../logical/CalcPruneAggregateCallRuleTest.scala | 31 +-
.../CalcPythonCorrelateTransposeRuleTest.scala | 7 +-
.../rules/logical/CalcRankTransposeRuleTest.scala | 29 +-
.../rules/logical/ConvertToNotInOrInRuleTest.scala | 25 +-
.../logical/CorrelateSortToRankRuleTest.scala | 14 +-
.../logical/DecomposeGroupingSetsRuleTest.scala | 5 +-
.../logical/ExpressionReductionRulesTest.scala | 5 +-
...AggregateExpandDistinctAggregatesRuleTest.scala | 6 +-
.../FlinkAggregateJoinTransposeRuleTest.scala | 22 +-
.../logical/FlinkAggregateRemoveRuleTest.scala | 40 +-
.../rules/logical/FlinkCalcMergeRuleTest.scala | 25 +-
.../rules/logical/FlinkFilterJoinRuleTest.scala | 13 +-
.../logical/FlinkJoinPushExpressionsRuleTest.scala | 8 +-
.../logical/FlinkJoinToMultiJoinRuleTest.scala | 8 +-
.../rules/logical/FlinkLimit0RemoveRuleTest.scala | 9 +-
.../FlinkLogicalRankRuleForConstantRangeTest.scala | 5 +-
.../FlinkLogicalRankRuleForRangeEndTest.scala | 5 +-
.../rules/logical/FlinkPruneEmptyRulesTest.scala | 8 +-
.../FlinkSemiAntiJoinFilterTransposeRuleTest.scala | 12 +-
.../FlinkSemiAntiJoinJoinTransposeRuleTest.scala | 14 +-
...FlinkSemiAntiJoinProjectTransposeRuleTest.scala | 15 +-
.../JoinConditionEqualityTransferRuleTest.scala | 5 +-
.../logical/JoinConditionTypeCoerceRuleTest.scala | 8 +-
.../JoinDependentConditionDerivationRuleTest.scala | 15 +-
.../logical/JoinDeriveNullFilterRuleTest.scala | 50 +-
...lCorrelateToJoinFromTemporalTableRuleTest.scala | 108 +-
.../plan/rules/logical/LogicalUnnestRuleTest.scala | 5 +-
.../ProjectPruneAggregateCallRuleTest.scala | 19 +-
.../ProjectSemiAntiJoinTransposeRuleTest.scala | 7 +-
...ojectWindowTableFunctionTransposeRuleTest.scala | 36 +-
.../logical/PruneAggregateCallRuleTestBase.scala | 7 +-
...shFilterIntoLegacyTableSourceScanRuleTest.scala | 16 +-
...ushLimitIntoLegacyTableSourceScanRuleTest.scala | 10 +-
...artitionIntoLegacyTableSourceScanRuleTest.scala | 35 +-
...hProjectIntoLegacyTableSourceScanRuleTest.scala | 15 +-
.../rules/logical/PythonCalcSplitRuleTest.scala | 11 +-
.../logical/PythonCorrelateSplitRuleTest.scala | 7 +-
.../rules/logical/PythonMapMergeRuleTest.scala | 18 +-
.../logical/RankNumberColumnRemoveRuleTest.scala | 5 +-
.../logical/RemoveSingleAggregateRuleTest.scala | 5 +-
.../ReplaceIntersectWithSemiJoinRuleTest.scala | 5 +-
.../logical/ReplaceMinusWithAntiJoinRuleTest.scala | 5 +-
.../logical/RewriteIntersectAllRuleTest.scala | 5 +-
.../rules/logical/RewriteMinusAllRuleTest.scala | 5 +-
.../RewriteMultiJoinConditionRuleTest.scala | 58 +-
.../logical/SimplifyFilterConditionRuleTest.scala | 10 +-
.../logical/SimplifyJoinConditionRuleTest.scala | 4 +-
.../rules/logical/SplitAggregateRuleTest.scala | 20 +-
...SplitPythonConditionFromCorrelateRuleTest.scala | 11 +-
.../SplitPythonConditionFromJoinRuleTest.scala | 11 +-
.../TemporalJoinRewriteWithUniqueKeyRuleTest.scala | 161 +-
.../rules/logical/WindowGroupReorderRuleTest.scala | 4 +-
.../subquery/FlinkRewriteSubQueryRuleTest.scala | 43 +-
.../logical/subquery/SubQueryAntiJoinTest.scala | 9 +-
.../logical/subquery/SubQuerySemiJoinTest.scala | 17 +-
.../rules/logical/subquery/SubQueryTestBase.scala | 7 +-
.../SubqueryCorrelateVariablesValidationTest.scala | 34 +-
.../batch/EnforceLocalAggRuleTestBase.scala | 2 -
.../batch/EnforceLocalHashAggRuleTest.scala | 45 +-
.../batch/EnforceLocalSortAggRuleTest.scala | 47 +-
.../RemoveRedundantLocalHashAggRuleTest.scala | 23 +-
.../batch/RemoveRedundantLocalRankRuleTest.scala | 4 +-
.../RemoveRedundantLocalSortAggRuleTest.scala | 18 +-
.../stream/ChangelogModeInferenceTest.scala | 277 ++-
...xpandWindowTableFunctionTransposeRuleTest.scala | 76 +-
...signerChangelogNormalizeTransposeRuleTest.scala | 110 +-
.../plan/schema/TimeIndicatorRelDataTypeTest.scala | 5 +-
.../planner/plan/stats/ValueIntervalTest.scala | 24 +-
.../table/planner/plan/stream/sql/CalcTest.scala | 5 +-
.../plan/stream/sql/DagOptimizationTest.scala | 201 +-
.../planner/plan/stream/sql/DeduplicateTest.scala | 18 +-
.../plan/stream/sql/FilterableSourceTest.scala | 39 +-
.../planner/plan/stream/sql/LegacySinkTest.scala | 67 +-
.../plan/stream/sql/LegacyTableSourceTest.scala | 215 +-
.../table/planner/plan/stream/sql/LimitTest.scala | 8 +-
.../plan/stream/sql/LimitableSourceTest.scala | 5 +-
.../plan/stream/sql/MatchRecognizeTest.scala | 1 -
.../stream/sql/MiniBatchIntervalInferTest.scala | 236 +-
.../plan/stream/sql/ModifiedMonotonicityTest.scala | 24 +-
.../plan/stream/sql/PartitionableSinkTest.scala | 1 -
.../plan/stream/sql/PartitionableSourceTest.scala | 46 +-
.../table/planner/plan/stream/sql/RankTest.scala | 245 +-
.../stream/sql/RelTimeIndicatorConverterTest.scala | 30 +-
.../planner/plan/stream/sql/SetOperatorsTest.scala | 4 +-
.../planner/plan/stream/sql/SortLimitTest.scala | 8 +-
.../table/planner/plan/stream/sql/SortTest.scala | 5 +-
.../plan/stream/sql/SourceWatermarkTest.scala | 148 +-
.../planner/plan/stream/sql/SubplanReuseTest.scala | 84 +-
.../planner/plan/stream/sql/TableScanTest.scala | 830 ++++---
.../planner/plan/stream/sql/TableSinkTest.scala | 803 ++++---
.../planner/plan/stream/sql/TableSourceTest.scala | 3 +-
.../table/planner/plan/stream/sql/UnionTest.scala | 78 +-
.../table/planner/plan/stream/sql/ValuesTest.scala | 1 -
.../plan/stream/sql/WindowDeduplicateTest.scala | 52 +-
.../planner/plan/stream/sql/WindowRankTest.scala | 34 +-
.../plan/stream/sql/WindowTableFunctionTest.scala | 63 +-
.../plan/stream/sql/agg/AggregateTest.scala | 310 +--
.../stream/sql/agg/DistinctAggregateTest.scala | 11 +-
.../plan/stream/sql/agg/GroupWindowTest.scala | 100 +-
.../plan/stream/sql/agg/GroupingSetsTest.scala | 26 +-
.../stream/sql/agg/IncrementalAggregateTest.scala | 5 +-
.../plan/stream/sql/agg/OverAggregateTest.scala | 17 +-
.../stream/sql/agg/TwoStageAggregateTest.scala | 11 +-
.../plan/stream/sql/agg/WindowAggregateTest.scala | 183 +-
.../plan/stream/sql/join/IntervalJoinTest.scala | 98 +-
.../planner/plan/stream/sql/join/JoinTest.scala | 292 ++-
.../plan/stream/sql/join/LookupJoinTest.scala | 340 +--
.../plan/stream/sql/join/SemiAntiJoinTest.scala | 1 -
.../stream/sql/join/TemporalFunctionJoinTest.scala | 35 +-
.../plan/stream/sql/join/TemporalJoinTest.scala | 428 ++--
.../plan/stream/sql/join/WindowJoinTest.scala | 111 +-
.../planner/plan/stream/table/AggregateTest.scala | 36 +-
.../table/planner/plan/stream/table/CalcTest.scala | 36 +-
.../plan/stream/table/ColumnFunctionsTest.scala | 44 +-
.../planner/plan/stream/table/CorrelateTest.scala | 38 +-
.../table/GroupWindowTableAggregateTest.scala | 11 +-
.../plan/stream/table/GroupWindowTest.scala | 170 +-
.../table/planner/plan/stream/table/JoinTest.scala | 83 +-
.../plan/stream/table/LegacyTableSourceTest.scala | 199 +-
.../plan/stream/table/OverAggregateTest.scala | 166 +-
.../plan/stream/table/PythonAggregateTest.scala | 23 +-
.../planner/plan/stream/table/PythonCalcTest.scala | 1 -
.../table/PythonGroupWindowAggregateTest.scala | 77 +-
.../table/PythonOverWindowAggregateTest.scala | 33 +-
.../stream/table/PythonTableAggregateTest.scala | 13 +-
.../plan/stream/table/SetOperatorsTest.scala | 9 +-
.../plan/stream/table/TableAggregateTest.scala | 11 +-
.../plan/stream/table/TableSourceTest.scala | 30 +-
.../table/TemporalTableFunctionJoinTest.scala | 71 +-
.../plan/stream/table/TwoStageAggregateTest.scala | 11 +-
.../stringexpr/AggregateStringExpressionTest.scala | 59 +-
.../stringexpr/CalcStringExpressionTest.scala | 25 +-
.../stringexpr/CorrelateStringExpressionTest.scala | 32 +-
.../GroupWindowStringExpressionTest.scala | 98 +-
...pWindowTableAggregateStringExpressionTest.scala | 76 +-
.../OverWindowStringExpressionTest.scala | 109 +-
.../SetOperatorsStringExpressionTest.scala | 1 -
.../TableAggregateStringExpressionTest.scala | 10 +-
.../table/validation/AggregateValidationTest.scala | 22 +-
.../table/validation/CalcValidationTest.scala | 76 +-
.../table/validation/CorrelateValidationTest.scala | 59 +-
.../GroupWindowTableAggregateValidationTest.scala | 15 +-
.../validation/GroupWindowValidationTest.scala | 97 +-
.../validation/LegacyTableSinkValidationTest.scala | 36 +-
.../validation/OverWindowValidationTest.scala | 63 +-
.../validation/SetOperatorsValidationTest.scala | 5 +-
.../validation/TableAggregateValidationTest.scala | 29 +-
.../TemporalTableJoinValidationTest.scala | 32 +-
.../validation/UnsupportedOpsValidationTest.scala | 1 -
.../plan/trait/FlinkRelDistributionTest.scala | 36 +-
.../plan/utils/ColumnIntervalUtilTest.scala | 39 +-
.../planner/plan/utils/FlinkRelMdUtilTest.scala | 1 -
.../planner/plan/utils/FlinkRelOptUtilTest.scala | 72 +-
.../planner/plan/utils/FlinkRexUtilTest.scala | 412 ++--
.../planner/plan/utils/InputTypeBuilder.scala | 1 -
.../plan/utils/NestedProjectionUtilTest.scala | 106 +-
.../planner/plan/utils/PartitionPrunerTest.scala | 32 +-
.../planner/plan/utils/RexNodeExtractorTest.scala | 245 +-
.../planner/plan/utils/RexNodeRewriterTest.scala | 65 +-
.../table/planner/plan/utils/RexNodeTestBase.scala | 39 +-
.../plan/utils/TestContextTableFactory.scala | 14 +-
.../table/planner/plan/utils/lookupFunctions.scala | 40 +-
.../flink/table/planner/plan/utils/pojos.scala | 6 +-
.../planner/runtime/FileSystemITCaseBase.scala | 124 +-
.../batch/sql/BatchFileSystemITCaseBase.scala | 19 +-
.../planner/runtime/batch/sql/CalcITCase.scala | 1180 ++++++----
.../runtime/batch/sql/CodeSplitITCase.scala | 41 +-
.../runtime/batch/sql/CorrelateITCase.scala | 99 +-
.../runtime/batch/sql/CorrelateITCase2.scala | 66 +-
.../planner/runtime/batch/sql/DecimalITCase.scala | 76 +-
.../batch/sql/FileSystemTestCsvITCase.scala | 5 +-
.../runtime/batch/sql/LegacyLimitITCase.scala | 50 +-
.../batch/sql/LegacyTableSourceITCase.scala | 183 +-
.../runtime/batch/sql/Limit0RemoveITCase.scala | 1 -
.../planner/runtime/batch/sql/LimitITCase.scala | 3 +-
.../planner/runtime/batch/sql/MiscITCase.scala | 116 +-
.../runtime/batch/sql/MultipleInputITCase.scala | 33 +-
.../runtime/batch/sql/OverAggregateITCase.scala | 810 ++++---
.../batch/sql/PartitionableSinkITCase.scala | 199 +-
.../batch/sql/PartitionableSourceITCase.scala | 134 +-
.../planner/runtime/batch/sql/RankITCase.scala | 191 +-
.../runtime/batch/sql/SetOperatorsITCase.scala | 31 +-
.../runtime/batch/sql/SortLimitITCase.scala | 60 +-
.../runtime/batch/sql/TableScanITCase.scala | 22 +-
.../runtime/batch/sql/TableSinkITCase.scala | 51 +-
.../runtime/batch/sql/TableSourceITCase.scala | 253 ++-
.../runtime/batch/sql/TimestampITCase.scala | 204 +-
.../planner/runtime/batch/sql/UnionITCase.scala | 46 +-
.../planner/runtime/batch/sql/UnnestITCase.scala | 105 +-
.../planner/runtime/batch/sql/ValuesITCase.scala | 1 -
.../batch/sql/WindowTableFunctionITCase.scala | 1019 +++++++--
.../batch/sql/agg/AggregateITCaseBase.scala | 321 ++-
.../sql/agg/AggregateJoinTransposeITCase.scala | 115 +-
.../sql/agg/AggregateReduceGroupingITCase.scala | 597 +++--
.../batch/sql/agg/AggregateRemoveITCase.scala | 211 +-
.../sql/agg/DistinctAggregateITCaseBase.scala | 88 +-
.../runtime/batch/sql/agg/GroupWindowITCase.scala | 332 ++-
.../runtime/batch/sql/agg/GroupingSetsITCase.scala | 332 ++-
.../runtime/batch/sql/agg/HashAggITCase.scala | 11 +-
.../sql/agg/HashDistinctAggregateITCase.scala | 8 +-
.../batch/sql/agg/PruneAggregateCallITCase.scala | 7 +-
.../runtime/batch/sql/agg/SortAggITCase.scala | 97 +-
.../sql/agg/SortDistinctAggregateITCase.scala | 8 +-
.../runtime/batch/sql/join/InnerJoinITCase.scala | 9 +-
.../sql/join/JoinConditionTypeCoerceITCase.scala | 155 +-
.../runtime/batch/sql/join/JoinITCase.scala | 620 +++--
.../runtime/batch/sql/join/JoinITCaseHelper.scala | 5 +-
.../batch/sql/join/JoinWithoutKeyITCase.scala | 220 +-
.../runtime/batch/sql/join/LookupJoinITCase.scala | 132 +-
.../runtime/batch/sql/join/OuterJoinITCase.scala | 245 +-
.../runtime/batch/sql/join/ScalarQueryITCase.scala | 9 +-
.../runtime/batch/sql/join/SemiJoinITCase.scala | 81 +-
.../runtime/batch/table/AggregationITCase.scala | 212 +-
.../planner/runtime/batch/table/CalcITCase.scala | 175 +-
.../runtime/batch/table/CorrelateITCase.scala | 83 +-
.../runtime/batch/table/DecimalITCase.scala | 80 +-
.../runtime/batch/table/GroupWindowITCase.scala | 213 +-
.../planner/runtime/batch/table/JoinITCase.scala | 219 +-
.../runtime/batch/table/LegacyLimitITCase.scala | 21 +-
.../batch/table/LegacyTableSinkITCase.scala | 144 +-
.../planner/runtime/batch/table/LimitITCase.scala | 2 +-
.../runtime/batch/table/OverAggregateITCase.scala | 399 ++--
.../runtime/batch/table/SetOperatorsITCase.scala | 34 +-
.../planner/runtime/batch/table/SortITCase.scala | 29 +-
.../runtime/batch/table/TableSinkITCase.scala | 529 +++--
...AbstractTwoInputStreamOperatorWithTTLTest.scala | 26 +-
.../harness/GroupAggregateHarnessTest.scala | 16 +-
.../planner/runtime/harness/HarnessTestBase.scala | 42 +-
.../runtime/harness/OverAggregateHarnessTest.scala | 762 +++----
.../planner/runtime/harness/RankHarnessTest.scala | 21 +-
.../harness/TableAggregateHarnessTest.scala | 29 +-
.../harness/WindowAggregateHarnessTest.scala | 426 +++-
...WindowAggregateUseDaylightTimeHarnessTest.scala | 97 +-
.../harness/WindowTableFunctionHarnessTest.scala | 881 +++++---
.../runtime/stream/FsStreamingSinkITCaseBase.scala | 86 +-
.../runtime/stream/sql/AggregateITCase.scala | 631 +++---
.../runtime/stream/sql/AggregateRemoveITCase.scala | 226 +-
.../runtime/stream/sql/AsyncLookupJoinITCase.scala | 112 +-
.../planner/runtime/stream/sql/CalcITCase.scala | 249 +-
.../runtime/stream/sql/ChangelogSourceITCase.scala | 296 ++-
.../runtime/stream/sql/CorrelateITCase.scala | 100 +-
.../runtime/stream/sql/DataStreamScalaITCase.scala | 31 +-
.../runtime/stream/sql/DeduplicateITCase.scala | 136 +-
.../stream/sql/FilterableSourceITCase.scala | 18 +-
.../stream/sql/FsStreamingSinkTestCsvITCase.scala | 5 +-
.../runtime/stream/sql/GroupWindowITCase.scala | 165 +-
.../runtime/stream/sql/IntervalJoinITCase.scala | 176 +-
.../planner/runtime/stream/sql/JoinITCase.scala | 397 +++-
.../stream/sql/LegacyTableSourceITCase.scala | 195 +-
.../runtime/stream/sql/Limit0RemoveITCase.scala | 3 +-
.../planner/runtime/stream/sql/LimitITCase.scala | 15 +-
.../runtime/stream/sql/LimitableSourceITCase.scala | 19 +-
.../runtime/stream/sql/LookupJoinITCase.scala | 176 +-
.../runtime/stream/sql/MatchRecognizeITCase.scala | 112 +-
.../runtime/stream/sql/OverAggregateITCase.scala | 439 ++--
.../stream/sql/PartitionableSourceITCase.scala | 48 +-
.../stream/sql/PruneAggregateCallITCase.scala | 4 +-
.../planner/runtime/stream/sql/RankITCase.scala | 430 ++--
.../stream/sql/SemiAntiJoinStreamITCase.scala | 75 +-
.../runtime/stream/sql/SetOperatorsITCase.scala | 13 +-
.../planner/runtime/stream/sql/SortITCase.scala | 41 +-
.../runtime/stream/sql/SortLimitITCase.scala | 25 +-
.../runtime/stream/sql/SourceWatermarkITCase.scala | 71 +-
.../runtime/stream/sql/SplitAggregateITCase.scala | 312 +--
.../stream/sql/StreamFileSystemITCaseBase.scala | 24 +-
.../stream/sql/StreamFileSystemTestCsvITCase.scala | 5 +-
.../stream/sql/StreamTableEnvironmentITCase.scala | 93 +-
.../runtime/stream/sql/TableScanITCase.scala | 31 +-
.../runtime/stream/sql/TableSinkITCase.scala | 230 +-
.../runtime/stream/sql/TableSourceITCase.scala | 120 +-
.../runtime/stream/sql/TemporalJoinITCase.scala | 520 ++---
.../runtime/stream/sql/TemporalSortITCase.scala | 88 +-
.../sql/TemporalTableFunctionJoinITCase.scala | 40 +-
.../runtime/stream/sql/TimeAttributeITCase.scala | 75 +-
.../runtime/stream/sql/TimestampITCase.scala | 80 +-
.../planner/runtime/stream/sql/UnnestITCase.scala | 53 +-
.../planner/runtime/stream/sql/ValuesITCase.scala | 5 +-
.../runtime/stream/sql/WindowAggregateITCase.scala | 92 +-
.../stream/sql/WindowDeduplicateITCase.scala | 52 +-
.../stream/sql/WindowDistinctAggregateITCase.scala | 191 +-
.../runtime/stream/sql/WindowJoinITCase.scala | 88 +-
.../runtime/stream/sql/WindowRankITCase.scala | 91 +-
.../stream/sql/WindowTableFunctionITCase.scala | 60 +-
.../runtime/stream/table/AggregateITCase.scala | 132 +-
.../planner/runtime/stream/table/CalcITCase.scala | 228 +-
.../runtime/stream/table/CorrelateITCase.scala | 77 +-
.../runtime/stream/table/GroupWindowITCase.scala | 101 +-
.../table/GroupWindowTableAggregateITCase.scala | 60 +-
.../planner/runtime/stream/table/JoinITCase.scala | 611 +++--
.../stream/table/LegacyTableSinkITCase.scala | 295 ++-
.../runtime/stream/table/OverAggregateITCase.scala | 282 ++-
.../runtime/stream/table/RetractionITCase.scala | 44 +-
.../runtime/stream/table/SetOperatorsITCase.scala | 37 +-
.../runtime/stream/table/SubQueryITCase.scala | 26 +-
.../stream/table/TableAggregateITCase.scala | 95 +-
.../runtime/stream/table/TableSinkITCase.scala | 1377 +++++------
.../stream/table/TableToDataStreamITCase.scala | 139 +-
.../planner/runtime/utils/BatchTableEnvUtil.scala | 305 ++-
.../planner/runtime/utils/BatchTestBase.scala | 262 ++-
.../runtime/utils/CollectionBatchExecTable.scala | 43 +-
.../utils/InMemoryLookupableTableSource.scala | 65 +-
.../planner/runtime/utils/SortTestUtils.scala | 52 +-
.../planner/runtime/utils/StreamTableEnvUtil.scala | 17 +-
.../planner/runtime/utils/StreamTestSink.scala | 90 +-
.../planner/runtime/utils/StreamingTestBase.scala | 13 +-
.../runtime/utils/StreamingWithAggTestBase.scala | 10 +-
.../utils/StreamingWithMiniBatchTestBase.scala | 4 +-
.../runtime/utils/StreamingWithStateTestBase.scala | 32 +-
.../table/planner/runtime/utils/TestData.scala | 660 ++++--
.../table/planner/runtime/utils/TestSinkUtil.scala | 11 +-
.../table/planner/runtime/utils/TimeTestUtil.scala | 17 +-
.../utils/UserDefinedFunctionTestUtils.scala | 36 +-
.../flink/table/planner/utils/AvgAggFunction.scala | 117 +-
.../table/planner/utils/CountAggFunction.scala | 6 +-
.../table/planner/utils/DateTimeTestUtil.scala | 1 -
.../planner/utils/LogicalPlanFormatUtils.scala | 10 +-
.../planner/utils/MemoryTableSourceSinkUtil.scala | 34 +-
.../flink/table/planner/utils/SumAggFunction.scala | 44 +-
.../flink/table/planner/utils/TableTestBase.scala | 692 +++---
.../utils/TestLegacyLimitableTableSource.scala | 24 +-
.../planner/utils/UserDefinedAggFunctions.scala | 27 +-
.../utils/UserDefinedTableAggFunctions.scala | 26 +-
.../planner/utils/UserDefinedTableFunctions.scala | 40 +-
.../table/planner/utils/testTableSourceSinks.scala | 401 ++--
.../api/scala/actions/CountCollectITCase.scala | 17 +-
.../PartitionOperatorTranslationTest.scala | 33 +-
.../BatchScalaAPICompletenessTest.scala | 252 ++-
.../ScalaAPICompletenessTestBase.scala | 29 +-
.../api/scala/functions/ClosureCleanerITCase.scala | 19 +-
.../scala/io/ScalaCsvReaderWithPOJOITCase.scala | 12 +-
.../manual/MassiveCaseClassSortingITCase.scala | 129 +-
.../api/scala/migration/MigrationTestTypes.scala | 1 -
.../migration/ScalaSerializersMigrationTest.scala | 12 +-
.../StatefulJobSavepointMigrationITCase.scala | 81 +-
...StatefulJobWBroadcastStateMigrationITCase.scala | 124 +-
.../api/scala/operators/AggregateITCase.scala | 17 +-
.../scala/operators/AggregateOperatorTest.scala | 13 +-
.../flink/api/scala/operators/CoGroupITCase.scala | 235 +-
.../api/scala/operators/CoGroupOperatorTest.scala | 61 +-
.../flink/api/scala/operators/CrossITCase.scala | 64 +-
.../flink/api/scala/operators/DistinctITCase.scala | 18 +-
.../api/scala/operators/DistinctOperatorTest.scala | 18 +-
.../flink/api/scala/operators/ExamplesITCase.scala | 23 +-
.../flink/api/scala/operators/FilterITCase.scala | 38 +-
.../flink/api/scala/operators/FirstNITCase.scala | 17 +-
.../api/scala/operators/FirstNOperatorTest.scala | 61 +-
.../flink/api/scala/operators/FlatMapITCase.scala | 71 +-
.../api/scala/operators/GroupCombineITCase.scala | 24 +-
.../api/scala/operators/GroupReduceITCase.scala | 761 +++----
.../flink/api/scala/operators/GroupingTest.scala | 34 +-
.../flink/api/scala/operators/JoinITCase.scala | 88 +-
.../api/scala/operators/JoinOperatorTest.scala | 36 +-
.../flink/api/scala/operators/MapITCase.scala | 32 +-
.../api/scala/operators/OuterJoinITCase.scala | 83 +-
.../api/scala/operators/PartitionITCase.scala | 84 +-
.../flink/api/scala/operators/ReduceITCase.scala | 89 +-
.../flink/api/scala/operators/SampleITCase.scala | 9 +-
.../api/scala/operators/SortPartitionITCase.scala | 62 +-
.../api/scala/operators/SumMinMaxITCase.scala | 21 +-
.../flink/api/scala/operators/UnionITCase.scala | 13 +-
.../translation/AggregateTranslationTest.scala | 16 +-
.../CoGroupCustomPartitioningTest.scala | 167 +-
.../CoGroupGroupSortTranslationTest.scala | 107 +-
...CustomPartitioningGroupingKeySelectorTest.scala | 102 +-
.../CustomPartitioningGroupingPojoTest.scala | 80 +-
.../CustomPartitioningGroupingTupleTest.scala | 184 +-
.../translation/CustomPartitioningTest.scala | 143 +-
.../DeltaIterationTranslationTest.scala | 52 +-
.../translation/JoinCustomPartitioningTest.scala | 167 +-
.../translation/PartitioningTestClasses.scala | 3 -
.../translation/ReduceTranslationTest.scala | 63 +-
.../CaseClassNormalizedKeySortingTest.scala | 69 +-
.../scala/runtime/ScalaSpecialTypesITCase.scala | 101 +-
.../flink/api/scala/util/CollectionDataSets.scala | 56 +-
.../flink/api/scala/util/DataSetUtilsITCase.scala | 7 +-
1316 files changed, 61547 insertions(+), 56386 deletions(-)
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala
index 52a6bed4870..ec6796d2e34 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.hadoop.mapred
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapred.HadoopInputFormatBase
+
import org.apache.hadoop.mapred.{InputFormat, JobConf}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
index b78665115e0..bfa65445248 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.hadoop.mapred
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapred.HadoopOutputFormatBase
+
import org.apache.hadoop.mapred.{JobConf, OutputCommitter, OutputFormat}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala
index 8efb53de1c0..3fa6666b201 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.scala.hadoop.mapreduce
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormatBase
+
import org.apache.hadoop.mapreduce.{InputFormat, Job}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala
index 8095304268b..23c35c8f9f8 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.scala.hadoop.mapreduce
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapreduce.HadoopOutputFormatBase
+
import org.apache.hadoop.mapreduce.{Job, OutputFormat}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala
index a59af64d22c..6275574b8ab 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala
@@ -15,32 +15,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.hadoopcompatibility.scala
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.scala.hadoop.{mapred, mapreduce}
+
import org.apache.hadoop.fs.{Path => HadoopPath}
-import org.apache.hadoop.mapred.{JobConf, FileInputFormat => MapredFileInputFormat, InputFormat => MapredInputFormat}
+import org.apache.hadoop.mapred.{FileInputFormat => MapredFileInputFormat, InputFormat => MapredInputFormat, JobConf}
+import org.apache.hadoop.mapreduce.{InputFormat => MapreduceInputFormat, Job}
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat => MapreduceFileInputFormat}
-import org.apache.hadoop.mapreduce.{Job, InputFormat => MapreduceInputFormat}
/**
- * HadoopInputs is a utility class to use Apache Hadoop InputFormats with Apache Flink.
- *
- * It provides methods to create Flink InputFormat wrappers for Hadoop
- * [[org.apache.hadoop.mapred.InputFormat]] and [[org.apache.hadoop.mapreduce.InputFormat]].
- *
- * Key value pairs produced by the Hadoop InputFormats are converted into [[Tuple2]] where
- * the first field is the key and the second field is the value.
- *
- */
+ * HadoopInputs is a utility class to use Apache Hadoop InputFormats with Apache Flink.
+ *
+ * It provides methods to create Flink InputFormat wrappers for Hadoop
+ * [[org.apache.hadoop.mapred.InputFormat]] and [[org.apache.hadoop.mapreduce.InputFormat]].
+ *
+ * Key value pairs produced by the Hadoop InputFormats are converted into [[Tuple2]] where the first
+ * field is the key and the second field is the value.
+ */
object HadoopInputs {
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapred.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapred.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapredInputFormat: MapredFileInputFormat[K, V],
key: Class[K],
@@ -55,9 +54,9 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapred.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapred.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapredInputFormat: MapredFileInputFormat[K, V],
key: Class[K],
@@ -68,26 +67,24 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that reads a Hadoop sequence
- * file with the given key and value classes.
- */
- def readSequenceFile[K, V](
- key: Class[K],
- value: Class[V],
- inputPath: String)(implicit tpe: TypeInformation[(K, V)]): mapred.HadoopInputFormat[K, V] = {
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that reads a Hadoop sequence
+ * file with the given key and value classes.
+ */
+ def readSequenceFile[K, V](key: Class[K], value: Class[V], inputPath: String)(implicit
+ tpe: TypeInformation[(K, V)]): mapred.HadoopInputFormat[K, V] = {
readHadoopFile(
new org.apache.hadoop.mapred.SequenceFileInputFormat[K, V],
key,
value,
inputPath
- )
+ )
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapred.InputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapred.InputFormat]].
+ */
def createHadoopInput[K, V](
mapredInputFormat: MapredInputFormat[K, V],
key: Class[K],
@@ -98,9 +95,9 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapreduceInputFormat: MapreduceFileInputFormat[K, V],
key: Class[K],
@@ -115,22 +112,22 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapreduceInputFormat: MapreduceFileInputFormat[K, V],
key: Class[K],
value: Class[V],
- inputPath: String)(implicit tpe: TypeInformation[(K, V)]): mapreduce.HadoopInputFormat[K, V] =
- {
+ inputPath: String)(implicit
+ tpe: TypeInformation[(K, V)]): mapreduce.HadoopInputFormat[K, V] = {
readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance)
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapreduce.InputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapreduce.InputFormat]].
+ */
def createHadoopInput[K, V](
mapreduceInputFormat: MapreduceInputFormat[K, V],
key: Class[K],
@@ -140,4 +137,3 @@ object HadoopInputs {
new mapreduce.HadoopInputFormat[K, V](mapreduceInputFormat, key, value, job)
}
}
-
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala
index df5f36152b6..b54fef4f1a1 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala
@@ -20,6 +20,7 @@ package org.apache.flink.api.hadoopcompatibility.scala
import org.apache.flink.api.java.typeutils.WritableTypeInfo
import org.apache.flink.api.scala._
import org.apache.flink.util.TestLogger
+
import org.apache.hadoop.io.Text
import org.junit.Assert._
import org.junit.Test
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala
index 1bf470c9ab7..ce4115b13a7 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala
@@ -23,6 +23,7 @@ import org.apache.flink.hadoopcompatibility.scala.HadoopInputs
import org.apache.flink.test.testdata.WordCountData
import org.apache.flink.test.util.{JavaProgramTestBase, TestBaseUtils}
import org.apache.flink.util.OperatingSystem
+
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{LongWritable, Text}
import org.apache.hadoop.mapred.{FileOutputFormat, JobConf, TextInputFormat, TextOutputFormat}
@@ -38,31 +39,34 @@ class WordCountMapredITCase extends JavaProgramTestBase {
Assume.assumeTrue("This test can't run successfully on Windows.", !OperatingSystem.isWindows)
}
- protected override def preSubmit() {
+ override protected def preSubmit() {
textPath = createTempFile("text.txt", WordCountData.TEXT)
resultPath = getTempDirPath("result")
}
- protected override def postSubmit() {
- TestBaseUtils.compareResultsByLinesInMemory(WordCountData.COUNTS,
- resultPath, Array[String](".", "_"))
+ override protected def postSubmit() {
+ TestBaseUtils.compareResultsByLinesInMemory(
+ WordCountData.COUNTS,
+ resultPath,
+ Array[String](".", "_"))
}
- private def internalRun (): Unit = {
+ private def internalRun(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val input =
- env.createInput(HadoopInputs.readHadoopFile(new TextInputFormat, classOf[LongWritable],
- classOf[Text], textPath))
+ env.createInput(
+ HadoopInputs
+ .readHadoopFile(new TextInputFormat, classOf[LongWritable], classOf[Text], textPath))
val counts = input
.map(_._2.toString)
- .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map( (_, 1)))
+ .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map((_, 1)))
.groupBy(0)
.sum(1)
val words = counts
- .map( t => (new Text(t._1), new LongWritable(t._2)) )
+ .map(t => (new Text(t._1), new LongWritable(t._2)))
val hadoopOutputFormat = new HadoopOutputFormat[Text, LongWritable](
new TextOutputFormat[Text, LongWritable],
@@ -81,4 +85,3 @@ class WordCountMapredITCase extends JavaProgramTestBase {
postSubmit()
}
}
-
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala
index d8d0646e362..597e0164259 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.hadoopcompatibility.scala
import org.apache.flink.api.scala._
@@ -24,6 +23,7 @@ import org.apache.flink.hadoopcompatibility.scala.HadoopInputs
import org.apache.flink.test.testdata.WordCountData
import org.apache.flink.test.util.{JavaProgramTestBase, TestBaseUtils}
import org.apache.flink.util.OperatingSystem
+
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{LongWritable, Text}
import org.apache.hadoop.mapreduce.Job
@@ -41,14 +41,16 @@ class WordCountMapreduceITCase extends JavaProgramTestBase {
Assume.assumeTrue("This test can't run successfully on Windows.", !OperatingSystem.isWindows)
}
- protected override def preSubmit() {
+ override protected def preSubmit() {
textPath = createTempFile("text.txt", WordCountData.TEXT)
resultPath = getTempDirPath("result")
}
- protected override def postSubmit() {
- TestBaseUtils.compareResultsByLinesInMemory(WordCountData.COUNTS,
- resultPath, Array[String](".", "_"))
+ override protected def postSubmit() {
+ TestBaseUtils.compareResultsByLinesInMemory(
+ WordCountData.COUNTS,
+ resultPath,
+ Array[String](".", "_"))
}
protected def testProgram() {
@@ -56,26 +58,26 @@ class WordCountMapreduceITCase extends JavaProgramTestBase {
postSubmit()
}
- private def internalRun (): Unit = {
+ private def internalRun(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val input =
- env.createInput(HadoopInputs.readHadoopFile(new TextInputFormat, classOf[LongWritable],
- classOf[Text], textPath))
+ env.createInput(
+ HadoopInputs
+ .readHadoopFile(new TextInputFormat, classOf[LongWritable], classOf[Text], textPath))
val counts = input
.map(_._2.toString)
- .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map( (_, 1)))
+ .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map((_, 1)))
.groupBy(0)
.sum(1)
val words = counts
- .map( t => (new Text(t._1), new LongWritable(t._2)) )
+ .map(t => (new Text(t._1), new LongWritable(t._2)))
val job = Job.getInstance()
- val hadoopOutputFormat = new HadoopOutputFormat[Text, LongWritable](
- new TextOutputFormat[Text, LongWritable],
- job)
+ val hadoopOutputFormat =
+ new HadoopOutputFormat[Text, LongWritable](new TextOutputFormat[Text, LongWritable], job)
hadoopOutputFormat.getConfiguration.set("mapred.textoutputformat.separator", " ")
FileOutputFormat.setOutputPath(job, new Path(resultPath))
@@ -85,4 +87,3 @@ class WordCountMapreduceITCase extends JavaProgramTestBase {
env.execute("Hadoop Compat WordCount")
}
}
-
diff --git a/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala b/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala
index 64918156e56..d19fd01e78a 100644
--- a/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala
+++ b/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala
@@ -15,28 +15,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.hcatalog.scala
import org.apache.flink.configuration
import org.apache.flink.hcatalog.HCatInputFormatBase
+
import org.apache.hadoop.conf.Configuration
import org.apache.hive.hcatalog.data.HCatRecord
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema
/**
- * A InputFormat to read from HCatalog tables.
- * The InputFormat supports projection (selection and order of fields) and partition filters.
- *
- * Data can be returned as [[HCatRecord]] or Scala tuples.
- * Scala tuples support only up to 22 fields.
+ * A InputFormat to read from HCatalog tables. The InputFormat supports projection (selection and
+ * order of fields) and partition filters.
*
+ * Data can be returned as [[HCatRecord]] or Scala tuples. Scala tuples support only up to 22
+ * fields.
*/
class HCatInputFormat[T](
- database: String,
- table: String,
- config: Configuration
- ) extends HCatInputFormatBase[T](database, table, config) {
+ database: String,
+ table: String,
+ config: Configuration
+) extends HCatInputFormatBase[T](database, table, config) {
def this(database: String, table: String) {
this(database, table, new Configuration)
@@ -57,97 +56,87 @@ class HCatInputFormat[T](
var i: Int = 0
while (i < this.fieldNames.length) {
- val o: AnyRef = record.get(this.fieldNames(i), this.outputSchema)
-
- // partition columns are returned as String
- // Check and convert to actual type.
- this.outputSchema.get(i).getType match {
- case HCatFieldSchema.Type.INT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toInt
- }
- else {
- vals(i) = o.asInstanceOf[Int]
- }
- case HCatFieldSchema.Type.TINYINT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toInt.toByte
- }
- else {
- vals(i) = o.asInstanceOf[Byte]
- }
- case HCatFieldSchema.Type.SMALLINT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toInt.toShort
- }
- else {
- vals(i) = o.asInstanceOf[Short]
- }
- case HCatFieldSchema.Type.BIGINT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toLong
- }
- else {
- vals(i) = o.asInstanceOf[Long]
- }
- case HCatFieldSchema.Type.BOOLEAN =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toBoolean
- }
- else {
- vals(i) = o.asInstanceOf[Boolean]
- }
- case HCatFieldSchema.Type.FLOAT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toFloat
- }
- else {
- vals(i) = o.asInstanceOf[Float]
- }
- case HCatFieldSchema.Type.DOUBLE =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toDouble
- }
- else {
- vals(i) = o.asInstanceOf[Double]
- }
- case HCatFieldSchema.Type.STRING =>
- vals(i) = o
- case HCatFieldSchema.Type.BINARY =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type BINARY.")
- }
- else {
- vals(i) = o.asInstanceOf[Array[Byte]]
- }
- case HCatFieldSchema.Type.ARRAY =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type ARRAY.")
- }
- else {
- vals(i) = o.asInstanceOf[List[Object]]
- }
- case HCatFieldSchema.Type.MAP =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type MAP.")
- }
- else {
- vals(i) = o.asInstanceOf[Map[Object, Object]]
- }
- case HCatFieldSchema.Type.STRUCT =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type STRUCT.")
- }
- else {
- vals(i) = o.asInstanceOf[List[Object]]
- }
- case _ =>
- throw new RuntimeException("Invalid type " + this.outputSchema.get(i).getType +
- " encountered.")
- }
+ val o: AnyRef = record.get(this.fieldNames(i), this.outputSchema)
- i += 1
+ // partition columns are returned as String
+ // Check and convert to actual type.
+ this.outputSchema.get(i).getType match {
+ case HCatFieldSchema.Type.INT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toInt
+ } else {
+ vals(i) = o.asInstanceOf[Int]
+ }
+ case HCatFieldSchema.Type.TINYINT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toInt.toByte
+ } else {
+ vals(i) = o.asInstanceOf[Byte]
+ }
+ case HCatFieldSchema.Type.SMALLINT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toInt.toShort
+ } else {
+ vals(i) = o.asInstanceOf[Short]
+ }
+ case HCatFieldSchema.Type.BIGINT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toLong
+ } else {
+ vals(i) = o.asInstanceOf[Long]
+ }
+ case HCatFieldSchema.Type.BOOLEAN =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toBoolean
+ } else {
+ vals(i) = o.asInstanceOf[Boolean]
+ }
+ case HCatFieldSchema.Type.FLOAT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toFloat
+ } else {
+ vals(i) = o.asInstanceOf[Float]
+ }
+ case HCatFieldSchema.Type.DOUBLE =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toDouble
+ } else {
+ vals(i) = o.asInstanceOf[Double]
+ }
+ case HCatFieldSchema.Type.STRING =>
+ vals(i) = o
+ case HCatFieldSchema.Type.BINARY =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type BINARY.")
+ } else {
+ vals(i) = o.asInstanceOf[Array[Byte]]
+ }
+ case HCatFieldSchema.Type.ARRAY =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type ARRAY.")
+ } else {
+ vals(i) = o.asInstanceOf[List[Object]]
+ }
+ case HCatFieldSchema.Type.MAP =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type MAP.")
+ } else {
+ vals(i) = o.asInstanceOf[Map[Object, Object]]
+ }
+ case HCatFieldSchema.Type.STRUCT =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type STRUCT.")
+ } else {
+ vals(i) = o.asInstanceOf[List[Object]]
+ }
+ case _ =>
+ throw new RuntimeException(
+ "Invalid type " + this.outputSchema.get(i).getType +
+ " encountered.")
}
+
+ i += 1
+ }
createScalaTuple(vals)
}
@@ -172,58 +161,247 @@ class HCatInputFormat[T](
new Tuple8(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7))
.asInstanceOf[T]
case 9 =>
- new Tuple9(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8)).asInstanceOf[T]
+ new Tuple9(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7), vals(8))
+ .asInstanceOf[T]
case 10 =>
- new Tuple10(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9)).asInstanceOf[T]
+ new Tuple10(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9)).asInstanceOf[T]
case 11 =>
- new Tuple11(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10)).asInstanceOf[T]
+ new Tuple11(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10)).asInstanceOf[T]
case 12 =>
- new Tuple12(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11)).asInstanceOf[T]
+ new Tuple12(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11)).asInstanceOf[T]
case 13 =>
- new Tuple13(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12)).asInstanceOf[T]
+ new Tuple13(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12)).asInstanceOf[T]
case 14 =>
- new Tuple14(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13)).asInstanceOf[T]
+ new Tuple14(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13)).asInstanceOf[T]
case 15 =>
- new Tuple15(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14)).asInstanceOf[T]
+ new Tuple15(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14)).asInstanceOf[T]
case 16 =>
- new Tuple16(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15))
+ new Tuple16(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15))
.asInstanceOf[T]
case 17 =>
- new Tuple17(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
+ new Tuple17(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
vals(16)).asInstanceOf[T]
case 18 =>
- new Tuple18(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17)).asInstanceOf[T]
+ new Tuple18(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17)).asInstanceOf[T]
case 19 =>
- new Tuple19(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18)).asInstanceOf[T]
+ new Tuple19(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18)).asInstanceOf[T]
case 20 =>
- new Tuple20(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18), vals(19)).asInstanceOf[T]
+ new Tuple20(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18),
+ vals(19)).asInstanceOf[T]
case 21 =>
- new Tuple21(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18), vals(19), vals(20)).asInstanceOf[T]
+ new Tuple21(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18),
+ vals(19),
+ vals(20)).asInstanceOf[T]
case 22 =>
- new Tuple22(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18), vals(19), vals(20), vals(21)).asInstanceOf[T]
+ new Tuple22(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18),
+ vals(19),
+ vals(20),
+ vals(21)).asInstanceOf[T]
case _ =>
throw new RuntimeException("Only up to 22 fields supported for Scala Tuples.")
- }
+ }
}
}
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala b/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala
index 8af5eec65b7..46e7d8098a0 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala
@@ -34,13 +34,15 @@ object ScalaJob {
// we want to go through serialization to check for kryo issues
env.disableOperatorChaining()
- env.fromElements(new NonPojo()).map(new MapFunction[NonPojo, NonPojo] {
- override def map(value: NonPojo): NonPojo = {
- // use some method that was removed in 2.12+
- BoxesRunTime.hashFromNumber(value.getSomeInt)
- value
- }
- })
+ env
+ .fromElements(new NonPojo())
+ .map(new MapFunction[NonPojo, NonPojo] {
+ override def map(value: NonPojo): NonPojo = {
+ // use some method that was removed in 2.12+
+ BoxesRunTime.hashFromNumber(value.getSomeInt)
+ value
+ }
+ })
env.execute();
}
diff --git a/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala b/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala
index 74c7a13b16f..985b2a62296 100644
--- a/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala
+++ b/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala
@@ -1,12 +1,13 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.quickstarts.test
import org.apache.flink.api.connector.sink2.SinkWriter
@@ -22,28 +22,30 @@ import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.connector.elasticsearch.sink.{Elasticsearch7SinkBuilder, RequestIndexer}
import org.apache.flink.streaming.api.datastream.DataStream
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+
import org.apache.http.HttpHost
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.client.Requests
import scala.collection.JavaConversions.mapAsJavaMap
-
object Elasticsearch7SinkExample {
def main(args: Array[String]) {
val parameterTool = ParameterTool.fromArgs(args)
if (parameterTool.getNumberOfParameters < 3) {
- println("Missing parameters!\n" + "Usage:" +
- " --numRecords <numRecords> --index <index> --type <type>")
+ println(
+ "Missing parameters!\n" + "Usage:" +
+ " --numRecords <numRecords> --index <index> --type <type>")
return
}
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.enableCheckpointing(5000)
- val source: DataStream[(String)] = env.fromSequence(0, 20 - 1)
- .map(v => "message #" + v.toString)
+ val source: DataStream[(String)] = env
+ .fromSequence(0, 20 - 1)
+ .map(v => "message #" + v.toString)
source.sinkTo(
new Elasticsearch7SinkBuilder[String]
@@ -51,8 +53,9 @@ object Elasticsearch7SinkExample {
// be buffered
.setBulkFlushMaxActions(1)
.setHosts(new HttpHost("127.0.0.1", 9200, "http"))
- .setEmitter((element: String, context: SinkWriter.Context, indexer: RequestIndexer) =>
- indexer.add(createIndexRequest(element, parameterTool)))
+ .setEmitter(
+ (element: String, context: SinkWriter.Context, indexer: RequestIndexer) =>
+ indexer.add(createIndexRequest(element, parameterTool)))
.build())
env.execute("Elasticsearch7.x end to end sink test example")
@@ -64,7 +67,9 @@ object Elasticsearch7SinkExample {
"data" -> element.asInstanceOf[AnyRef]
)
- Requests.indexRequest.index(parameterTool.getRequired("index"))
- .`type`(parameterTool.getRequired("type")).source(mapAsJavaMap(json2))
+ Requests.indexRequest
+ .index(parameterTool.getRequired("index"))
+ .`type`(parameterTool.getRequired("type"))
+ .source(mapAsJavaMap(json2))
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala
index e5a16a51783..2d4623bc473 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala
@@ -29,46 +29,40 @@ import scala.collection.JavaConverters._
/**
* This example implements a basic K-Means clustering algorithm.
*
- * K-Means is an iterative clustering algorithm and works as follows:
- * K-Means is given a set of data points to be clustered and an initial set of ''K'' cluster
- * centers.
- * In each iteration, the algorithm computes the distance of each data point to each cluster center.
- * Each point is assigned to the cluster center which is closest to it.
- * Subsequently, each cluster center is moved to the center (''mean'') of all points that have
- * been assigned to it.
- * The moved cluster centers are fed into the next iteration.
- * The algorithm terminates after a fixed number of iterations (as in this implementation)
- * or if cluster centers do not (significantly) move in an iteration.
- * This is the Wikipedia entry for the [[http://en.wikipedia
- * .org/wiki/K-means_clustering K-Means Clustering algorithm]].
+ * K-Means is an iterative clustering algorithm and works as follows: K-Means is given a set of data
+ * points to be clustered and an initial set of ''K'' cluster centers. In each iteration, the
+ * algorithm computes the distance of each data point to each cluster center. Each point is assigned
+ * to the cluster center which is closest to it. Subsequently, each cluster center is moved to the
+ * center (''mean'') of all points that have been assigned to it. The moved cluster centers are fed
+ * into the next iteration. The algorithm terminates after a fixed number of iterations (as in this
+ * implementation) or if cluster centers do not (significantly) move in an iteration. This is the
+ * Wikipedia entry for the
+ * [[http://en.wikipedia .org/wiki/K-means_clustering K-Means Clustering algorithm]].
*
- * This implementation works on two-dimensional data points.
- * It computes an assignment of data points to cluster centers, i.e.,
- * each data point is annotated with the id of the final cluster (center) it belongs to.
+ * This implementation works on two-dimensional data points. It computes an assignment of data
+ * points to cluster centers, i.e., each data point is annotated with the id of the final cluster
+ * (center) it belongs to.
*
* Input files are plain text files and must be formatted as follows:
*
- * - Data points are represented as two double values separated by a blank character.
- * Data points are separated by newline characters.
- * For example `"1.2 2.3\n5.3 7.2\n"` gives two data points (x=1.2, y=2.3) and (x=5.3,
- * y=7.2).
- * - Cluster centers are represented by an integer id and a point value.
- * For example `"1 6.2 3.2\n2 2.9 5.7\n"` gives two centers (id=1, x=6.2,
- * y=3.2) and (id=2, x=2.9, y=5.7).
+ * - Data points are represented as two double values separated by a blank character. Data points
+ * are separated by newline characters. For example `"1.2 2.3\n5.3 7.2\n"` gives two data points
+ * (x=1.2, y=2.3) and (x=5.3, y=7.2).
+ * - Cluster centers are represented by an integer id and a point value. For example `"1 6.2
+ * 3.2\n2 2.9 5.7\n"` gives two centers (id=1, x=6.2, y=3.2) and (id=2, x=2.9, y=5.7).
*
* Usage:
* {{{
* KMeans --points <path> --centroids <path> --output <path> --iterations <n>
* }}}
* If no parameters are provided, the program is run with default data from
- * [[org.apache.flink.examples.java.clustering.util.KMeansData]]
- * and 10 iterations.
+ * [[org.apache.flink.examples.java.clustering.util.KMeansData]] and 10 iterations.
*
* This example shows how to use:
*
- * - Bulk iterations
- * - Broadcast variables in bulk iterations
- * - Scala case classes
+ * - Bulk iterations
+ * - Broadcast variables in bulk iterations
+ * - Scala case classes
*/
object KMeans {
@@ -85,14 +79,19 @@ object KMeans {
val points: DataSet[Point] = getPointDataSet(params, env)
val centroids: DataSet[Centroid] = getCentroidDataSet(params, env)
- val finalCentroids = centroids.iterate(params.getInt("iterations", 10)) { currentCentroids =>
- val newCentroids = points
- .map(new SelectNearestCenter).withBroadcastSet(currentCentroids, "centroids")
- .map { x => (x._1, x._2, 1L) }.withForwardedFields("_1; _2")
- .groupBy(0)
- .reduce { (p1, p2) => (p1._1, p1._2.add(p2._2), p1._3 + p2._3) }.withForwardedFields("_1")
- .map { x => new Centroid(x._1, x._2.div(x._3)) }.withForwardedFields("_1->id")
- newCentroids
+ val finalCentroids = centroids.iterate(params.getInt("iterations", 10)) {
+ currentCentroids =>
+ val newCentroids = points
+ .map(new SelectNearestCenter)
+ .withBroadcastSet(currentCentroids, "centroids")
+ .map(x => (x._1, x._2, 1L))
+ .withForwardedFields("_1; _2")
+ .groupBy(0)
+ .reduce((p1, p2) => (p1._1, p1._2.add(p2._2), p1._3 + p2._3))
+ .withForwardedFields("_1")
+ .map(x => new Centroid(x._1, x._2.div(x._3)))
+ .withForwardedFields("_1->id")
+ newCentroids
}
val clusteredPoints: DataSet[(Int, Point)] =
@@ -121,7 +120,7 @@ object KMeans {
} else {
println("Executing K-Means example with default centroid data set.")
println("Use --centroids to specify file input.")
- env.fromCollection(KMeansData.CENTROIDS map {
+ env.fromCollection(KMeansData.CENTROIDS.map {
case Array(id, x, y) =>
new Centroid(id.asInstanceOf[Int], x.asInstanceOf[Double], y.asInstanceOf[Double])
})
@@ -137,7 +136,7 @@ object KMeans {
} else {
println("Executing K-Means example with default points data set.")
println("Use --points to specify file input.")
- env.fromCollection(KMeansData.POINTS map {
+ env.fromCollection(KMeansData.POINTS.map {
case Array(x, y) => new Point(x.asInstanceOf[Double], y.asInstanceOf[Double])
})
}
@@ -148,9 +147,9 @@ object KMeans {
// *************************************************************************
/**
- * Common trait for operations supported by both points and centroids
- * Note: case class inheritance is not allowed in Scala
- */
+ * Common trait for operations supported by both points and centroids Note: case class inheritance
+ * is not allowed in Scala
+ */
trait Coordinate extends Serializable {
var x: Double
@@ -181,14 +180,10 @@ object KMeans {
}
- /**
- * A simple two-dimensional point.
- */
+ /** A simple two-dimensional point. */
case class Point(var x: Double = 0, var y: Double = 0) extends Coordinate
- /**
- * A simple two-dimensional centroid, basically a point with an ID.
- */
+ /** A simple two-dimensional centroid, basically a point with an ID. */
case class Centroid(var id: Int = 0, var x: Double = 0, var y: Double = 0) extends Coordinate {
def this(id: Int, p: Point) {
@@ -225,5 +220,3 @@ object KMeans {
}
}
-
-
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
index c7186123c0e..8f58c7af8d3 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.java.utils.ParameterTool
@@ -27,39 +26,36 @@ import org.apache.flink.util.Collector
* An implementation of the connected components algorithm, using a delta iteration.
*
* Initially, the algorithm assigns each vertex an unique ID. In each step, a vertex picks the
- * minimum of its own ID and its neighbors' IDs, as its new ID and tells its neighbors about its
- * new ID. After the algorithm has completed, all vertices in the same component will have the same
- * ID.
+ * minimum of its own ID and its neighbors' IDs, as its new ID and tells its neighbors about its new
+ * ID. After the algorithm has completed, all vertices in the same component will have the same ID.
*
- * A vertex whose component ID did not change needs not propagate its information in the next
- * step. Because of that, the algorithm is easily expressible via a delta iteration. We here model
- * the solution set as the vertices with their current component ids, and the workset as the changed
+ * A vertex whose component ID did not change needs not propagate its information in the next step.
+ * Because of that, the algorithm is easily expressible via a delta iteration. We here model the
+ * solution set as the vertices with their current component ids, and the workset as the changed
* vertices. Because we see all vertices initially as changed, the initial workset and the initial
* solution set are identical. Also, the delta to the solution set is consequently also the next
* workset.
- *
+ *
* Input files are plain text files and must be formatted as follows:
*
* - Vertices represented as IDs and separated by new-line characters. For example,
* `"1\n2\n12\n42\n63"` gives five vertices (1), (2), (12), (42), and (63).
* - Edges are represented as pairs for vertex IDs which are separated by space characters. Edges
- * are separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"`
- * gives four (undirected) edges (1)-(2), (2)-(12), (1)-(12), and (42)-(63).
+ * are separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"` gives four
+ * (undirected) edges (1)-(2), (2)-(12), (1)-(12), and (42)-(63).
*
* Usage:
* {{{
* ConnectedComponents --vertices <path> --edges <path> --result <path> --iterations <n>
* }}}
- *
+ *
* If no parameters are provided, the program is run with default data from
* [[org.apache.flink.examples.java.graph.util.ConnectedComponentsData]] and 10 iterations.
- *
*
* This example shows how to use:
*
* - Delta Iterations
- * - Generic-typed Functions
- *
+ * - Generic-typed Functions
*/
object ConnectedComponents {
@@ -78,30 +74,36 @@ object ConnectedComponents {
// read vertex and edge data
// assign the initial components (equal to the vertex id)
val vertices =
- getVertexDataSet(env, params).map { id => (id, id) }.withForwardedFields("*->_1;*->_2")
+ getVertexDataSet(env, params).map(id => (id, id)).withForwardedFields("*->_1;*->_2")
// undirected edges by emitting for each input edge the input
// edges itself and an inverted version
val edges =
- getEdgeDataSet(env, params).flatMap { edge => Seq(edge, (edge._2, edge._1)) }
+ getEdgeDataSet(env, params).flatMap(edge => Seq(edge, (edge._2, edge._1)))
// open a delta iteration
val verticesWithComponents = vertices.iterateDelta(vertices, maxIterations, Array("_1")) {
(s, ws) =>
-
// apply the step logic: join with the edges
- val allNeighbors = ws.join(edges).where(0).equalTo(0) { (vertex, edge) =>
- (edge._2, vertex._2)
- }.withForwardedFieldsFirst("_2->_2").withForwardedFieldsSecond("_2->_1")
+ val allNeighbors = ws
+ .join(edges)
+ .where(0)
+ .equalTo(0)((vertex, edge) => (edge._2, vertex._2))
+ .withForwardedFieldsFirst("_2->_2")
+ .withForwardedFieldsSecond("_2->_1")
// select the minimum neighbor
val minNeighbors = allNeighbors.groupBy(0).min(1)
// update if the component of the candidate is smaller
- val updatedComponents = minNeighbors.join(s).where(0).equalTo(0) {
- (newVertex, oldVertex, out: Collector[(Long, Long)]) =>
- if (newVertex._2 < oldVertex._2) out.collect(newVertex)
- }.withForwardedFieldsFirst("*")
+ val updatedComponents = minNeighbors
+ .join(s)
+ .where(0)
+ .equalTo(0) {
+ (newVertex, oldVertex, out: Collector[(Long, Long)]) =>
+ if (newVertex._2 < oldVertex._2) out.collect(newVertex)
+ }
+ .withForwardedFieldsFirst("*")
// delta and new workset are identical
(updatedComponents, updatedComponents)
@@ -119,31 +121,30 @@ object ConnectedComponents {
private def getVertexDataSet(env: ExecutionEnvironment, params: ParameterTool): DataSet[Long] = {
if (params.has("vertices")) {
- env.readCsvFile[Tuple1[Long]](
- params.get("vertices"),
- includedFields = Array(0))
- .map { x => x._1 }
- }
- else {
+ env
+ .readCsvFile[Tuple1[Long]](params.get("vertices"), includedFields = Array(0))
+ .map(x => x._1)
+ } else {
println("Executing ConnectedComponents example with default vertices data set.")
println("Use --vertices to specify file input.")
env.fromCollection(ConnectedComponentsData.VERTICES)
}
}
- private def getEdgeDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(Long, Long)] = {
+ private def getEdgeDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(Long, Long)] = {
if (params.has("edges")) {
- env.readCsvFile[(Long, Long)](
- params.get("edges"),
- fieldDelimiter = " ",
- includedFields = Array(0, 1))
- .map { x => (x._1, x._2)}
- }
- else {
+ env
+ .readCsvFile[(Long, Long)](
+ params.get("edges"),
+ fieldDelimiter = " ",
+ includedFields = Array(0, 1))
+ .map(x => (x._1, x._2))
+ } else {
println("Executing ConnectedComponents example with default edges data set.")
println("Use --edges to specify file input.")
- val edgeData = ConnectedComponentsData.EDGES map {
+ val edgeData = ConnectedComponentsData.EDGES.map {
case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
}
env.fromCollection(edgeData)
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
index 159891bad4c..2649890a2c4 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.scala._
@@ -23,11 +22,11 @@ import org.apache.flink.util.Collector
object DeltaPageRank {
- private final val DAMPENING_FACTOR: Double = 0.85
- private final val NUM_VERTICES = 5
- private final val INITIAL_RANK = 1.0 / NUM_VERTICES
- private final val RANDOM_JUMP = (1 - DAMPENING_FACTOR) / NUM_VERTICES
- private final val THRESHOLD = 0.0001 / NUM_VERTICES
+ final private val DAMPENING_FACTOR: Double = 0.85
+ final private val NUM_VERTICES = 5
+ final private val INITIAL_RANK = 1.0 / NUM_VERTICES
+ final private val RANDOM_JUMP = (1 - DAMPENING_FACTOR) / NUM_VERTICES
+ final private val THRESHOLD = 0.0001 / NUM_VERTICES
type Page = (Long, Double)
type Adjacency = (Long, Array[Long])
@@ -38,61 +37,66 @@ object DeltaPageRank {
val env = ExecutionEnvironment.getExecutionEnvironment
- val rawLines: DataSet[String] = env.fromElements(
- "1 2 3 4",
- "2 1",
- "3 5",
- "4 2 3",
- "5 2 4")
+ val rawLines: DataSet[String] = env.fromElements("1 2 3 4", "2 1", "3 5", "4 2 3", "5 2 4")
val adjacency: DataSet[Adjacency] = rawLines
- .map(str => {
- val elements = str.split(' ')
- val id = elements(0).toLong
- val neighbors = elements.slice(1, elements.length).map(_.toLong)
- (id, neighbors)
- })
-
- val initialRanks: DataSet[Page] = adjacency.flatMap {
- (adj, out: Collector[Page]) =>
- {
- val targets = adj._2
- val rankPerTarget = INITIAL_RANK * DAMPENING_FACTOR / targets.length
-
- // dampen fraction to targets
- for (target <- targets) {
- out.collect((target, rankPerTarget))
+ .map(
+ str => {
+ val elements = str.split(' ')
+ val id = elements(0).toLong
+ val neighbors = elements.slice(1, elements.length).map(_.toLong)
+ (id, neighbors)
+ })
+
+ val initialRanks: DataSet[Page] = adjacency
+ .flatMap {
+ (adj, out: Collector[Page]) =>
+ {
+ val targets = adj._2
+ val rankPerTarget = INITIAL_RANK * DAMPENING_FACTOR / targets.length
+
+ // dampen fraction to targets
+ for (target <- targets) {
+ out.collect((target, rankPerTarget))
+ }
+
+ // random jump to self
+ out.collect((adj._1, RANDOM_JUMP))
}
+ }
+ .groupBy(0)
+ .sum(1)
- // random jump to self
- out.collect((adj._1, RANDOM_JUMP))
- }
- }
- .groupBy(0).sum(1)
-
- val initialDeltas = initialRanks.map { (page) => (page._1, page._2 - INITIAL_RANK) }
- .withForwardedFields("_1")
+ val initialDeltas = initialRanks
+ .map((page) => (page._1, page._2 - INITIAL_RANK))
+ .withForwardedFields("_1")
val iteration = initialRanks.iterateDelta(initialDeltas, maxIterations, Array(0)) {
(solutionSet, workset) =>
{
- val deltas = workset.join(adjacency).where(0).equalTo(0) {
- (lastDeltas, adj, out: Collector[Page]) =>
- {
- val targets = adj._2
- val deltaPerTarget = DAMPENING_FACTOR * lastDeltas._2 / targets.length
-
- for (target <- targets) {
- out.collect((target, deltaPerTarget))
+ val deltas = workset
+ .join(adjacency)
+ .where(0)
+ .equalTo(0) {
+ (lastDeltas, adj, out: Collector[Page]) =>
+ {
+ val targets = adj._2
+ val deltaPerTarget = DAMPENING_FACTOR * lastDeltas._2 / targets.length
+
+ for (target <- targets) {
+ out.collect((target, deltaPerTarget))
+ }
}
- }
- }
- .groupBy(0).sum(1)
+ }
+ .groupBy(0)
+ .sum(1)
.filter(x => Math.abs(x._2) > THRESHOLD)
- val rankUpdates = solutionSet.join(deltas).where(0).equalTo(0) {
- (current, delta) => (current._1, current._2 + delta._2)
- }.withForwardedFieldsFirst("_1")
+ val rankUpdates = solutionSet
+ .join(deltas)
+ .where(0)
+ .equalTo(0)((current, delta) => (current._1, current._2 + delta._2))
+ .withForwardedFieldsFirst("_1")
(rankUpdates, deltas)
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala
index 32adeb59274..a91ad34087d 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.common.functions.GroupReduceFunction
@@ -30,40 +29,32 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
/**
- * Triangle enumeration is a pre-processing step to find closely connected parts in graphs.
- * A triangle consists of three edges that connect three vertices with each other.
- *
- * The algorithm works as follows:
- * It groups all edges that share a common vertex and builds triads, i.e., triples of vertices
- * that are connected by two edges. Finally, all triads are filtered for which no third edge exists
- * that closes the triangle.
- *
+ * Triangle enumeration is a pre-processing step to find closely connected parts in graphs. A
+ * triangle consists of three edges that connect three vertices with each other.
+ *
+ * The algorithm works as follows: It groups all edges that share a common vertex and builds triads,
+ * i.e., triples of vertices that are connected by two edges. Finally, all triads are filtered for
+ * which no third edge exists that closes the triangle.
+ *
* Input files are plain text files and must be formatted as follows:
*
- * - Edges are represented as pairs for vertex IDs which are separated by space
- * characters. Edges are separated by new-line characters.
- * For example `"1 2\n2 12\n1 12\n42 63"` gives four (undirected) edges (1)-(2), (2)-(12),
- * (1)-(12), and (42)-(63) that include a triangle
+ * - Edges are represented as pairs for vertex IDs which are separated by space characters. Edges
+ * are separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"` gives four
+ * (undirected) edges (1)-(2), (2)-(12), (1)-(12), and (42)-(63) that include a triangle
+ *
+ * <pre> (1) / \ (2)-(12) </pre>
*
- * <pre>
- * (1)
- * / \
- * (2)-(12)
- * </pre>
- *
- * Usage:
+ * Usage:
* {{{
* EnumTriangleBasic <edge path> <result path>
* }}}
- * <br>
- * If no parameters are provided, the program is run with default data from
+ * <br> If no parameters are provided, the program is run with default data from
* [[org.apache.flink.examples.java.graph.util.EnumTrianglesData]]
- *
- * This example shows how to use:
*
- * - Custom Java objects which extend Tuple
- * - Group Sorting
+ * This example shows how to use:
*
+ * - Custom Java objects which extend Tuple
+ * - Group Sorting
*/
object EnumTriangles {
@@ -92,17 +83,21 @@ object EnumTriangles {
}
env.fromCollection(edges)
}
-
+
// project edges by vertex id
- val edgesById = edges map(e => if (e.v1 < e.v2) e else Edge(e.v2, e.v1) )
-
+ val edgesById = edges.map(e => if (e.v1 < e.v2) e else Edge(e.v2, e.v1))
+
val triangles = edgesById
- // build triads
- .groupBy("v1").sortGroup("v2", Order.ASCENDING).reduceGroup(new TriadBuilder())
- // filter triads
- .join(edgesById).where("v2", "v3").equalTo("v1", "v2") { (t, _) => t }
- .withForwardedFieldsFirst("*")
-
+ // build triads
+ .groupBy("v1")
+ .sortGroup("v2", Order.ASCENDING)
+ .reduceGroup(new TriadBuilder())
+ // filter triads
+ .join(edgesById)
+ .where("v2", "v3")
+ .equalTo("v1", "v2")((t, _) => t)
+ .withForwardedFieldsFirst("*")
+
// emit result
if (params.has("output")) {
triangles.writeAsCsv(params.get("output"), "\n", ",")
@@ -112,7 +107,6 @@ object EnumTriangles {
println("Printing result to stdout. Use --output to specify output path.")
triangles.print()
}
-
}
@@ -122,32 +116,31 @@ object EnumTriangles {
case class Edge(v1: Int, v2: Int) extends Serializable
case class Triad(v1: Int, v2: Int, v3: Int) extends Serializable
-
-
+
// *************************************************************************
// USER FUNCTIONS
// *************************************************************************
/**
- * Builds triads (triples of vertices) from pairs of edges that share a vertex. The first vertex
- * of a triad is the shared vertex, the second and third vertex are ordered by vertexId. Assumes
- * that input edges share the first vertex and are in ascending order of the second vertex.
+ * Builds triads (triples of vertices) from pairs of edges that share a vertex. The first vertex
+ * of a triad is the shared vertex, the second and third vertex are ordered by vertexId. Assumes
+ * that input edges share the first vertex and are in ascending order of the second vertex.
*/
@ForwardedFields(Array("v1->v1"))
class TriadBuilder extends GroupReduceFunction[Edge, Triad] {
val vertices = mutable.MutableList[Integer]()
-
+
override def reduce(edges: java.lang.Iterable[Edge], out: Collector[Triad]) = {
-
+
// clear vertex list
vertices.clear()
// build and emit triads
- for(e <- edges.asScala) {
-
+ for (e <- edges.asScala) {
+
// combine vertex with all previously read vertices
- for(v <- vertices) {
+ for (v <- vertices) {
out.collect(Triad(e.v1, v, e.v2))
}
vertices += e.v2
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
index 4692dbd0fb7..afc1a44f1a1 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
@@ -17,8 +17,6 @@
*/
package org.apache.flink.examples.scala.graph
-import java.lang.Iterable
-
import org.apache.flink.api.common.functions.GroupReduceFunction
import org.apache.flink.api.java.aggregation.Aggregations.SUM
import org.apache.flink.api.java.utils.ParameterTool
@@ -26,11 +24,13 @@ import org.apache.flink.api.scala._
import org.apache.flink.examples.java.graph.util.PageRankData
import org.apache.flink.util.Collector
+import java.lang.Iterable
+
import scala.collection.JavaConverters._
/**
* A basic implementation of the Page Rank algorithm using a bulk iteration.
- *
+ *
* This implementation requires a set of pages and a set of directed links as input and works as
* follows.
*
@@ -40,16 +40,16 @@ import scala.collection.JavaConverters._
* with the new ranks of all pages. This implementation terminates after a fixed number of
* iterations. This is the Wikipedia entry for the
* [[http://en.wikipedia.org/wiki/Page_rank Page Rank algorithm]]
- *
+ *
* Input files are plain text files and must be formatted as follows:
*
- * - Pages represented as an (long) ID separated by new-line characters.
- * For example `"1\n2\n12\n42\n63"` gives five pages with IDs 1, 2, 12, 42, and 63.
- * - Links are represented as pairs of page IDs which are separated by space characters. Links
- * are separated by new-line characters.
- * For example `"1 2\n2 12\n1 12\n42 63"` gives four (directed) links (1)->(2), (2)->(12),
- * (1)->(12), and (42)->(63). For this simple implementation it is required that each page has
- * at least one incoming and one outgoing link (a page can point to itself).
+ * - Pages represented as an (long) ID separated by new-line characters. For example
+ * `"1\n2\n12\n42\n63"` gives five pages with IDs 1, 2, 12, 42, and 63.
+ * - Links are represented as pairs of page IDs which are separated by space characters. Links are
+ * separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"` gives four
+ * (directed) links (1)->(2), (2)->(12), (1)->(12), and (42)->(63). For this simple
+ * implementation it is required that each page has at least one incoming and one outgoing link
+ * (a page can point to itself).
*
* Usage:
* {{{
@@ -58,18 +58,17 @@ import scala.collection.JavaConverters._
*
* If no parameters are provided, the program is run with default data from
* [[org.apache.flink.examples.java.graph.util.PageRankData]] and 10 iterations.
- *
- * This example shows how to use:
*
- * - Bulk Iterations
- * - Default Join
- * - Configure user-defined functions using constructor parameters.
+ * This example shows how to use:
*
+ * - Bulk Iterations
+ * - Default Join
+ * - Configure user-defined functions using constructor parameters.
*/
object PageRankBasic {
- private final val DAMPENING_FACTOR: Double = 0.85
- private final val EPSILON: Double = 0.0001
+ final private val DAMPENING_FACTOR: Double = 0.85
+ final private val EPSILON: Double = 0.0001
def main(args: Array[String]) {
@@ -91,10 +90,11 @@ object PageRankBasic {
// build adjacency list from link input
val adjacencyLists = links
- .groupBy("sourceId").reduceGroup( new GroupReduceFunction[Link, AdjacencyList] {
+ .groupBy("sourceId")
+ .reduceGroup(new GroupReduceFunction[Link, AdjacencyList] {
override def reduce(values: Iterable[Link], out: Collector[AdjacencyList]): Unit = {
var outputId = -1L
- val outputList = values.asScala map { t => outputId = t.sourceId; t.targetId }
+ val outputList = values.asScala.map { t => outputId = t.sourceId; t.targetId }
out.collect(new AdjacencyList(outputId, outputList.toArray))
}
})
@@ -104,18 +104,22 @@ object PageRankBasic {
currentRanks =>
val newRanks = currentRanks
// distribute ranks to target pages
- .join(adjacencyLists).where("pageId").equalTo("sourceId") {
+ .join(adjacencyLists)
+ .where("pageId")
+ .equalTo("sourceId") {
(page, adjacent, out: Collector[Page]) =>
val targets = adjacent.targetIds
val len = targets.length
- adjacent.targetIds foreach { t => out.collect(Page(t, page.rank /len )) }
+ adjacent.targetIds.foreach(t => out.collect(Page(t, page.rank / len)))
}
// collect ranks and sum them up
- .groupBy("pageId").aggregate(SUM, "rank")
+ .groupBy("pageId")
+ .aggregate(SUM, "rank")
// apply dampening factor
- .map { p =>
- Page(p.pageId, (p.rank * DAMPENING_FACTOR) + ((1 - DAMPENING_FACTOR) / numPages))
- }.withForwardedFields("pageId")
+ .map {
+ p => Page(p.pageId, (p.rank * DAMPENING_FACTOR) + ((1 - DAMPENING_FACTOR) / numPages))
+ }
+ .withForwardedFields("pageId")
// terminate if no rank update was significant
val termination = currentRanks.join(newRanks).where("pageId").equalTo("pageId") {
@@ -153,8 +157,9 @@ object PageRankBasic {
// UTIL METHODS
// *************************************************************************
- private def getPagesDataSet(env: ExecutionEnvironment, params: ParameterTool):
- (DataSet[Long], Long) = {
+ private def getPagesDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): (DataSet[Long], Long) = {
if (params.has("pages") && params.has("numPages")) {
val pages = env
.readCsvFile[Tuple1[Long]](params.get("pages"), fieldDelimiter = " ", lineDelimiter = "\n")
@@ -167,16 +172,15 @@ object PageRankBasic {
}
}
- private def getLinksDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[Link] = {
+ private def getLinksDataSet(env: ExecutionEnvironment, params: ParameterTool): DataSet[Link] = {
if (params.has("links")) {
- env.readCsvFile[Link](params.get("links"), fieldDelimiter = " ",
- includedFields = Array(0, 1))
+ env.readCsvFile[Link](params.get("links"), fieldDelimiter = " ", includedFields = Array(0, 1))
} else {
println("Executing PageRank example with default links data set.")
println("Use --links to specify file input.")
- val edges = PageRankData.EDGES.map { case Array(v1, v2) => Link(v1.asInstanceOf[Long],
- v2.asInstanceOf[Long])}
+ val edges = PageRankData.EDGES.map {
+ case Array(v1, v2) => Link(v1.asInstanceOf[Long], v2.asInstanceOf[Long])
+ }
env.fromCollection(edges)
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
index 8f87b91bbed..c20abf15f86 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.java.utils.ParameterTool
@@ -25,7 +24,7 @@ import org.apache.flink.util.Collector
object TransitiveClosureNaive {
- def main (args: Array[String]): Unit = {
+ def main(args: Array[String]): Unit = {
val params: ParameterTool = ParameterTool.fromArgs(args)
@@ -37,15 +36,16 @@ object TransitiveClosureNaive {
val edges =
if (params.has("edges")) {
- env.readCsvFile[(Long, Long)](
- filePath = params.get("edges"),
- fieldDelimiter = " ",
- includedFields = Array(0, 1))
- .map { x => (x._1, x._2)}
+ env
+ .readCsvFile[(Long, Long)](
+ filePath = params.get("edges"),
+ fieldDelimiter = " ",
+ includedFields = Array(0, 1))
+ .map(x => (x._1, x._2))
} else {
println("Executing TransitiveClosure example with default edges data set.")
println("Use --edges to specify file input.")
- val edgeData = ConnectedComponentsData.EDGES map {
+ val edgeData = ConnectedComponentsData.EDGES.map {
case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
}
env.fromCollection(edgeData)
@@ -53,30 +53,35 @@ object TransitiveClosureNaive {
val maxIterations = params.getInt("iterations", 10)
- val paths = edges.iterateWithTermination(maxIterations) { prevPaths: DataSet[(Long, Long)] =>
-
- val nextPaths = prevPaths
- .join(edges)
- .where(1).equalTo(0) {
- (left, right) => (left._1,right._2)
- }.withForwardedFieldsFirst("_1").withForwardedFieldsSecond("_2")
- .union(prevPaths)
- .groupBy(0, 1)
- .reduce((l, r) => l).withForwardedFields("_1; _2")
+ val paths = edges.iterateWithTermination(maxIterations) {
+ prevPaths: DataSet[(Long, Long)] =>
+ val nextPaths = prevPaths
+ .join(edges)
+ .where(1)
+ .equalTo(0)((left, right) => (left._1, right._2))
+ .withForwardedFieldsFirst("_1")
+ .withForwardedFieldsSecond("_2")
+ .union(prevPaths)
+ .groupBy(0, 1)
+ .reduce((l, r) => l)
+ .withForwardedFields("_1; _2")
- val terminate = prevPaths
- .coGroup(nextPaths)
- .where(0).equalTo(0) {
- (
- prev: Iterator[(Long, Long)],
- next: Iterator[(Long, Long)],
- out: Collector[(Long, Long)]) => {
- val prevPaths = prev.toSet
- for (n <- next)
- if (!prevPaths.contains(n)) out.collect(n)
- }
- }.withForwardedFieldsSecond("*")
- (nextPaths, terminate)
+ val terminate = prevPaths
+ .coGroup(nextPaths)
+ .where(0)
+ .equalTo(0) {
+ (
+ prev: Iterator[(Long, Long)],
+ next: Iterator[(Long, Long)],
+ out: Collector[(Long, Long)]) =>
+ {
+ val prevPaths = prev.toSet
+ for (n <- next)
+ if (!prevPaths.contains(n)) out.collect(n)
+ }
+ }
+ .withForwardedFieldsSecond("*")
+ (nextPaths, terminate)
}
if (params.has("output")) {
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala
index 3453ee8bf8b..ee5396f21bb 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.misc
import org.apache.flink.api.scala._
@@ -31,18 +30,20 @@ object PiEstimation {
// count how many of the samples would randomly fall into
// the upper right quadrant of the unit circle
val count =
- env.generateSequence(1, numSamples)
- .map { sample =>
- val x = Math.random()
- val y = Math.random()
- if (x * x + y * y < 1) 1L else 0L
+ env
+ .generateSequence(1, numSamples)
+ .map {
+ sample =>
+ val x = Math.random()
+ val y = Math.random()
+ if (x * x + y * y < 1) 1L else 0L
}
.reduce(_ + _)
// ratio of samples in upper right quadrant vs total samples gives surface of upper
// right quadrant, times 4 gives surface of whole unit circle, i.e. PI
val pi = count
- .map ( _ * 4.0 / numSamples)
+ .map(_ * 4.0 / numSamples)
println("We estimate Pi to be:")
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala
index d510a3630ac..c38e1876b7f 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.relational
import org.apache.flink.api.java.aggregation.Aggregations
@@ -23,72 +22,74 @@ import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.api.scala._
/**
- * This program implements a modified version of the TPC-H query 10.
- *
+ * This program implements a modified version of the TPC-H query 10.
+ *
* The original query can be found at
- * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf)
- * (page 45).
+ * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf) (page
+ * 45).
*
* This program implements the following SQL equivalent:
*
* {{{
- * SELECT
+ * SELECT
* c_custkey,
- * c_name,
+ * c_name,
* c_address,
- * n_name,
+ * n_name,
* c_acctbal
- * SUM(l_extendedprice * (1 - l_discount)) AS revenue,
- * FROM
- * customer,
- * orders,
- * lineitem,
- * nation
- * WHERE
- * c_custkey = o_custkey
- * AND l_orderkey = o_orderkey
- * AND YEAR(o_orderdate) > '1990'
- * AND l_returnflag = 'R'
- * AND c_nationkey = n_nationkey
- * GROUP BY
- * c_custkey,
- * c_name,
- * c_acctbal,
- * n_name,
+ * SUM(l_extendedprice * (1 - l_discount)) AS revenue,
+ * FROM
+ * customer,
+ * orders,
+ * lineitem,
+ * nation
+ * WHERE
+ * c_custkey = o_custkey
+ * AND l_orderkey = o_orderkey
+ * AND YEAR(o_orderdate) > '1990'
+ * AND l_returnflag = 'R'
+ * AND c_nationkey = n_nationkey
+ * GROUP BY
+ * c_custkey,
+ * c_name,
+ * c_acctbal,
+ * n_name,
* c_address
* }}}
*
- * Compared to the original TPC-H query this version does not print
- * c_phone and c_comment, only filters by years greater than 1990 instead of
- * a period of 3 months, and does not sort the result by revenue..
+ * Compared to the original TPC-H query this version does not print c_phone and c_comment, only
+ * filters by years greater than 1990 instead of a period of 3 months, and does not sort the result
+ * by revenue..
*
- * Input files are plain text CSV files using the pipe character ('|') as field separator
- * as generated by the TPC-H data generator which is available at
- * [http://www.tpc.org/tpch/](a href="http://www.tpc.org/tpch/).
+ * Input files are plain text CSV files using the pipe character ('|') as field separator as
+ * generated by the TPC-H data generator which is available at [http://www.tpc.org/tpch/](a
+ * href="http://www.tpc.org/tpch/).
*
- * Usage:
+ * Usage:
* {{{
- *TPCHQuery10 --customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>
+ * TPCHQuery10 --customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>
* }}}
- *
+ *
* This example shows how to use:
- * - tuple data types
- * - build-in aggregation functions
- * - join with size hints
- *
+ * - tuple data types
+ * - build-in aggregation functions
+ * - join with size hints
*/
object TPCHQuery10 {
def main(args: Array[String]) {
val params: ParameterTool = ParameterTool.fromArgs(args)
- if (!params.has("lineitem") && !params.has("customer") &&
- !params.has("orders") && !params.has("nation")) {
+ if (
+ !params.has("lineitem") && !params.has("customer") &&
+ !params.has("orders") && !params.has("nation")
+ ) {
println(" This program expects data from the TPC-H benchmark as input data.")
println(" Due to legal restrictions, we can not ship generated data.")
println(" You can find the TPC-H data generator at http://www.tpc.org/tpch/.")
- println(" Usage: TPCHQuery10" +
- "--customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>")
+ println(
+ " Usage: TPCHQuery10" +
+ "--customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>")
return
}
@@ -98,34 +99,44 @@ object TPCHQuery10 {
// make parameters available in the web interface
env.getConfig.setGlobalJobParameters(params)
- // get customer data set: (custkey, name, address, nationkey, acctbal)
+ // get customer data set: (custkey, name, address, nationkey, acctbal)
val customers = getCustomerDataSet(env, params.get("customer"))
// get orders data set: (orderkey, custkey, orderdate)
val orders = getOrdersDataSet(env, params.get("orders"))
// get lineitem data set: (orderkey, extendedprice, discount, returnflag)
val lineitems = getLineitemDataSet(env, params.get("lineitem"))
- // get nation data set: (nationkey, name)
+ // get nation data set: (nationkey, name)
val nations = getNationDataSet(env, params.get("nation"))
// filter orders by years
- val orders1990 = orders.filter( o => o._3.substring(0,4).toInt > 1990)
- .map( o => (o._1, o._2))
-
+ val orders1990 = orders
+ .filter(o => o._3.substring(0, 4).toInt > 1990)
+ .map(o => (o._1, o._2))
+
// filter lineitems by return status
- val lineitemsReturn = lineitems.filter( l => l._4.equals("R"))
- .map( l => (l._1, l._2 * (1 - l._3)) )
+ val lineitemsReturn = lineitems
+ .filter(l => l._4.equals("R"))
+ .map(l => (l._1, l._2 * (1 - l._3)))
// compute revenue by customer
- val revenueByCustomer = orders1990.joinWithHuge(lineitemsReturn).where(0).equalTo(0)
- .apply( (o,l) => (o._2, l._2) )
- .groupBy(0)
- .aggregate(Aggregations.SUM, 1)
+ val revenueByCustomer = orders1990
+ .joinWithHuge(lineitemsReturn)
+ .where(0)
+ .equalTo(0)
+ .apply((o, l) => (o._2, l._2))
+ .groupBy(0)
+ .aggregate(Aggregations.SUM, 1)
// compute final result by joining customer and nation information with revenue
- val result = customers.joinWithTiny(nations).where(3).equalTo(0)
- .apply( (c, n) => (c._1, c._2, c._3, n._2, c._5) )
- .join(revenueByCustomer).where(0).equalTo(0)
- .apply( (c, r) => (c._1, c._2, c._3, c._4, c._5, r._2) )
+ val result = customers
+ .joinWithTiny(nations)
+ .where(3)
+ .equalTo(0)
+ .apply((c, n) => (c._1, c._2, c._3, n._2, c._5))
+ .join(revenueByCustomer)
+ .where(0)
+ .equalTo(0)
+ .apply((c, r) => (c._1, c._2, c._3, c._4, c._5, r._2))
if (params.has("output")) {
// emit result
@@ -138,40 +149,41 @@ object TPCHQuery10 {
}
}
-
+
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
- private def getCustomerDataSet(env: ExecutionEnvironment, customerPath: String):
- DataSet[(Int, String, String, Int, Double)] = {
+
+ private def getCustomerDataSet(
+ env: ExecutionEnvironment,
+ customerPath: String): DataSet[(Int, String, String, Int, Double)] = {
env.readCsvFile[(Int, String, String, Int, Double)](
- customerPath,
- fieldDelimiter = "|",
- includedFields = Array(0,1,2,3,5) )
+ customerPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 1, 2, 3, 5))
}
-
- private def getOrdersDataSet(env: ExecutionEnvironment, ordersPath: String):
- DataSet[(Int, Int, String)] = {
+
+ private def getOrdersDataSet(
+ env: ExecutionEnvironment,
+ ordersPath: String): DataSet[(Int, Int, String)] = {
env.readCsvFile[(Int, Int, String)](
- ordersPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 1, 4) )
+ ordersPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 1, 4))
}
-
- private def getLineitemDataSet(env: ExecutionEnvironment, lineitemPath: String):
- DataSet[(Int, Double, Double, String)] = {
+
+ private def getLineitemDataSet(
+ env: ExecutionEnvironment,
+ lineitemPath: String): DataSet[(Int, Double, Double, String)] = {
env.readCsvFile[(Int, Double, Double, String)](
- lineitemPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 5, 6, 8) )
+ lineitemPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 5, 6, 8))
}
- private def getNationDataSet(env: ExecutionEnvironment, nationPath: String):
- DataSet[(Int, String)] = {
- env.readCsvFile[(Int, String)](
- nationPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 1) )
+ private def getNationDataSet(
+ env: ExecutionEnvironment,
+ nationPath: String): DataSet[(Int, String)] = {
+ env.readCsvFile[(Int, String)](nationPath, fieldDelimiter = "|", includedFields = Array(0, 1))
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala
index 2db1fe7f7bb..883ce048118 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.relational
import org.apache.flink.api.java.aggregation.Aggregations
@@ -23,51 +22,49 @@ import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.api.scala._
/**
- * This program implements a modified version of the TPC-H query 3. The
- * example demonstrates how to assign names to fields by extending the Tuple class.
- * The original query can be found at
- * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf)
- * (page 29).
+ * This program implements a modified version of the TPC-H query 3. The example demonstrates how to
+ * assign names to fields by extending the Tuple class. The original query can be found at
+ * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf) (page
+ * 29).
*
* This program implements the following SQL equivalent:
*
* {{{
- * SELECT
- * l_orderkey,
+ * SELECT
+ * l_orderkey,
* SUM(l_extendedprice*(1-l_discount)) AS revenue,
- * o_orderdate,
- * o_shippriority
- * FROM customer,
- * orders,
- * lineitem
+ * o_orderdate,
+ * o_shippriority
+ * FROM customer,
+ * orders,
+ * lineitem
* WHERE
- * c_mktsegment = '[SEGMENT]'
+ * c_mktsegment = '[SEGMENT]'
* AND c_custkey = o_custkey
* AND l_orderkey = o_orderkey
* AND o_orderdate < date '[DATE]'
* AND l_shipdate > date '[DATE]'
* GROUP BY
- * l_orderkey,
- * o_orderdate,
+ * l_orderkey,
+ * o_orderdate,
* o_shippriority;
* }}}
*
- * Compared to the original TPC-H query this version does not sort the result by revenue
- * and orderdate.
+ * Compared to the original TPC-H query this version does not sort the result by revenue and
+ * orderdate.
*
- * Input files are plain text CSV files using the pipe character ('|') as field separator
- * as generated by the TPC-H data generator which is available at
- * [http://www.tpc.org/tpch/](a href="http://www.tpc.org/tpch/).
+ * Input files are plain text CSV files using the pipe character ('|') as field separator as
+ * generated by the TPC-H data generator which is available at [http://www.tpc.org/tpch/](a
+ * href="http://www.tpc.org/tpch/).
*
- * Usage:
+ * Usage:
* {{{
* TPCHQuery3 --lineitem <path> --customer <path> --orders <path> --output <path>
* }}}
- *
+ *
* This example shows how to use:
- * - case classes and case class field addressing
- * - build-in aggregation functions
- *
+ * - case classes and case class field addressing
+ * - build-in aggregation functions
*/
object TPCHQuery3 {
@@ -78,8 +75,9 @@ object TPCHQuery3 {
println(" This program expects data from the TPC-H benchmark as input data.")
println(" Due to legal restrictions, we can not ship generated data.")
println(" You can find the TPC-H data generator at http://www.tpc.org/tpch/.")
- println(" Usage: TPCHQuery3 " +
- "--lineitem <path> --customer <path> --orders <path> [--output <path>]")
+ println(
+ " Usage: TPCHQuery3 " +
+ "--lineitem <path> --customer <path> --orders <path> [--output <path>]")
return
}
@@ -92,33 +90,37 @@ object TPCHQuery3 {
// set filter date
val dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd")
val date = dateFormat.parse("1995-03-12")
-
+
// read and filter lineitems by shipDate
val lineitems =
- getLineitemDataSet(env, params.get("lineitem")).
- filter( l => dateFormat.parse(l.shipDate).after(date) )
+ getLineitemDataSet(env, params.get("lineitem")).filter(
+ l => dateFormat.parse(l.shipDate).after(date))
// read and filter customers by market segment
val customers =
- getCustomerDataSet(env, params.get("customer")).
- filter( c => c.mktSegment.equals("AUTOMOBILE"))
+ getCustomerDataSet(env, params.get("customer")).filter(c => c.mktSegment.equals("AUTOMOBILE"))
// read orders
val orders = getOrdersDataSet(env, params.get("orders"))
- // filter orders by order date
- val items = orders.filter( o => dateFormat.parse(o.orderDate).before(date) )
- // filter orders by joining with customers
- .join(customers).where("custId").equalTo("custId").apply( (o,c) => o )
- // join with lineitems
- .join(lineitems).where("orderId").equalTo("orderId")
- .apply( (o,l) =>
- new ShippedItem( o.orderId,
- l.extdPrice * (1.0 - l.discount),
- o.orderDate,
- o.shipPrio ) )
+ // filter orders by order date
+ val items = orders
+ .filter(o => dateFormat.parse(o.orderDate).before(date))
+ // filter orders by joining with customers
+ .join(customers)
+ .where("custId")
+ .equalTo("custId")
+ .apply((o, c) => o)
+ // join with lineitems
+ .join(lineitems)
+ .where("orderId")
+ .equalTo("orderId")
+ .apply(
+ (o, l) =>
+ new ShippedItem(o.orderId, l.extdPrice * (1.0 - l.discount), o.orderDate, o.shipPrio))
// group by order and aggregate revenue
- val result = items.groupBy("orderId", "orderDate", "shipPrio")
- .aggregate(Aggregations.SUM, "revenue")
+ val result = items
+ .groupBy("orderId", "orderDate", "shipPrio")
+ .aggregate(Aggregations.SUM, "revenue")
if (params.has("output")) {
// emit result
@@ -131,11 +133,11 @@ object TPCHQuery3 {
}
}
-
+
// *************************************************************************
// USER DATA TYPES
// *************************************************************************
-
+
case class Lineitem(orderId: Long, extdPrice: Double, discount: Double, shipDate: String)
case class Order(orderId: Long, custId: Long, orderDate: String, shipPrio: Long)
case class Customer(custId: Long, mktSegment: String)
@@ -144,28 +146,23 @@ object TPCHQuery3 {
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
- private def getLineitemDataSet(env: ExecutionEnvironment, lineitemPath: String):
- DataSet[Lineitem] = {
+
+ private def getLineitemDataSet(
+ env: ExecutionEnvironment,
+ lineitemPath: String): DataSet[Lineitem] = {
env.readCsvFile[Lineitem](
- lineitemPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 5, 6, 10) )
+ lineitemPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 5, 6, 10))
}
- private def getCustomerDataSet(env: ExecutionEnvironment, customerPath: String):
- DataSet[Customer] = {
- env.readCsvFile[Customer](
- customerPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 6) )
+ private def getCustomerDataSet(
+ env: ExecutionEnvironment,
+ customerPath: String): DataSet[Customer] = {
+ env.readCsvFile[Customer](customerPath, fieldDelimiter = "|", includedFields = Array(0, 6))
}
- private def getOrdersDataSet(env: ExecutionEnvironment, ordersPath: String):
- DataSet[Order] = {
- env.readCsvFile[Order](
- ordersPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 1, 4, 7) )
+ private def getOrdersDataSet(env: ExecutionEnvironment, ordersPath: String): DataSet[Order] = {
+ env.readCsvFile[Order](ordersPath, fieldDelimiter = "|", includedFields = Array(0, 1, 4, 7))
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
index f068ed34727..f5e53c25293 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.relational
import org.apache.flink.api.java.utils.ParameterTool
@@ -24,8 +23,8 @@ import org.apache.flink.examples.java.relational.util.WebLogData
import org.apache.flink.util.Collector
/**
- * This program processes web logs and relational data.
- * It implements the following relational query:
+ * This program processes web logs and relational data. It implements the following relational
+ * query:
*
* {{{
* SELECT
@@ -44,11 +43,10 @@ import org.apache.flink.util.Collector
* );
* }}}
*
- *
- * Input files are plain text CSV files using the pipe character ('|') as field separator.
- * The tables referenced in the query can be generated using the
- * [org.apache.flink.examples.java.relational.util.WebLogDataGenerator]] and
- * have the following schemas
+ * Input files are plain text CSV files using the pipe character ('|') as field separator. The
+ * tables referenced in the query can be generated using the
+ * [org.apache.flink.examples.java.relational.util.WebLogDataGenerator]] and have the following
+ * schemas
*
* {{{
* CREATE TABLE Documents (
@@ -72,7 +70,6 @@ import org.apache.flink.util.Collector
* duration INT );
* }}}
*
- *
* Usage
* {{{
* WebLogAnalysis --documents <path> --ranks <path> --visits <path> --output <path>
@@ -83,10 +80,9 @@ import org.apache.flink.util.Collector
*
* This example shows how to use:
*
- * - tuple data types
- * - projection and join projection
- * - the CoGroup transformation for an anti-join
- *
+ * - tuple data types
+ * - projection and join projection
+ * - the CoGroup transformation for an anti-join
*/
object WebLogAnalysis {
@@ -113,17 +109,23 @@ object WebLogAnalysis {
val filteredVisits = visits
.filter(visit => visit._2.substring(0, 4).toInt == 2007)
- val joinDocsRanks = filteredDocs.join(filteredRanks).where(0).equalTo(1) {
- (doc, rank) => rank
- }.withForwardedFieldsSecond("*")
-
- val result = joinDocsRanks.coGroup(filteredVisits).where(1).equalTo(0) {
- (
- ranks: Iterator[(Int, String, Int)],
- visits: Iterator[(String, String)],
- out: Collector[(Int, String, Int)]) =>
+ val joinDocsRanks = filteredDocs
+ .join(filteredRanks)
+ .where(0)
+ .equalTo(1)((doc, rank) => rank)
+ .withForwardedFieldsSecond("*")
+
+ val result = joinDocsRanks
+ .coGroup(filteredVisits)
+ .where(1)
+ .equalTo(0) {
+ (
+ ranks: Iterator[(Int, String, Int)],
+ visits: Iterator[(String, String)],
+ out: Collector[(Int, String, Int)]) =>
if (visits.isEmpty) for (rank <- ranks) out.collect(rank)
- }.withForwardedFieldsFirst("*")
+ }
+ .withForwardedFieldsFirst("*")
// emit result
if (params.has("output")) {
@@ -136,8 +138,9 @@ object WebLogAnalysis {
}
- private def getDocumentsDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(String, String)] = {
+ private def getDocumentsDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(String, String)] = {
if (params.has("documents")) {
env.readCsvFile[(String, String)](
params.get("documents"),
@@ -146,15 +149,16 @@ object WebLogAnalysis {
} else {
println("Executing WebLogAnalysis example with default documents data set.")
println("Use --documents to specify file input.")
- val documents = WebLogData.DOCUMENTS map {
+ val documents = WebLogData.DOCUMENTS.map {
case Array(x, y) => (x.asInstanceOf[String], y.asInstanceOf[String])
}
env.fromCollection(documents)
}
}
- private def getRanksDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(Int, String, Int)] = {
+ private def getRanksDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(Int, String, Int)] = {
if (params.has("ranks")) {
env.readCsvFile[(Int, String, Int)](
params.get("ranks"),
@@ -163,15 +167,16 @@ object WebLogAnalysis {
} else {
println("Executing WebLogAnalysis example with default ranks data set.")
println("Use --ranks to specify file input.")
- val ranks = WebLogData.RANKS map {
+ val ranks = WebLogData.RANKS.map {
case Array(x, y, z) => (x.asInstanceOf[Int], y.asInstanceOf[String], z.asInstanceOf[Int])
}
env.fromCollection(ranks)
}
}
- private def getVisitsDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(String, String)] = {
+ private def getVisitsDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(String, String)] = {
if (params.has("visits")) {
env.readCsvFile[(String, String)](
params.get("visits"),
@@ -180,7 +185,7 @@ object WebLogAnalysis {
} else {
println("Executing WebLogAnalysis example with default visits data set.")
println("Use --visits to specify file input.")
- val visits = WebLogData.VISITS map {
+ val visits = WebLogData.VISITS.map {
case Array(x, y) => (x.asInstanceOf[String], y.asInstanceOf[String])
}
env.fromCollection(visits)
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala
index 0f3d6c8e1ff..2ad9e304a05 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.wordcount
import org.apache.flink.api.java.utils.ParameterTool
@@ -23,8 +22,8 @@ import org.apache.flink.api.scala._
import org.apache.flink.examples.java.wordcount.util.WordCountData
/**
- * Implements the "WordCount" program that computes a simple word occurrence histogram
- * over text files.
+ * Implements the "WordCount" program that computes a simple word occurrence histogram over text
+ * files.
*
* The input is a plain text file with lines separated by newline characters.
*
@@ -41,7 +40,6 @@ import org.apache.flink.examples.java.wordcount.util.WordCountData
* - write a simple Flink program.
* - use Tuple data types.
* - write and use user-defined functions.
- *
*/
object WordCount {
@@ -63,8 +61,9 @@ object WordCount {
env.fromCollection(WordCountData.WORDS)
}
- val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } }
- .map { (_, 1) }
+ val counts = text
+ .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty))
+ .map((_, 1))
.groupBy(0)
.sum(1)
@@ -78,5 +77,3 @@ object WordCount {
}
}
-
-
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala
index 3270980b16b..385f13b4ecd 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.async
import java.util.concurrent.ThreadLocalRandom
+
import scala.concurrent.{ExecutionContext, Future}
/** A simple asynchronous client that simulates interacting with an unreliable external service. */
@@ -26,7 +26,8 @@ class AsyncClient {
def query(key: Int)(implicit executor: ExecutionContext): Future[String] = Future {
val sleep = (ThreadLocalRandom.current.nextFloat * 100).toLong
- try Thread.sleep(sleep) catch {
+ try Thread.sleep(sleep)
+ catch {
case e: InterruptedException =>
throw new RuntimeException("AsyncClient was interrupted", e)
}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala
index b711a35c23c..b55cb60553c 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala
@@ -15,17 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.async
-
import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFunction}
import org.apache.flink.streaming.api.scala._
+import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFunction}
import org.apache.flink.streaming.examples.async.util.SimpleSource
import java.util.concurrent.TimeUnit
+
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.{Failure, Success}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala
index 4812ec41784..e51e977d35c 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala
@@ -15,12 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.iteration
import org.apache.flink.api.common.serialization.SimpleStringEncoder
-
-import java.util.Random
import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.api.scala._
import org.apache.flink.configuration.MemorySize
@@ -32,23 +29,23 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceCont
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import java.time.Duration
+import java.util.Random
/**
* Example illustrating iterations in Flink streaming.
*
- * The program sums up random numbers and counts additions
- * it performs to reach a specific threshold in an iterative streaming fashion.
+ * The program sums up random numbers and counts additions it performs to reach a specific threshold
+ * in an iterative streaming fashion.
*
* This example shows how to use:
*
- * - streaming iterations,
- * - buffer timeout to enhance latency,
- * - directed outputs.
- *
+ * - streaming iterations,
+ * - buffer timeout to enhance latency,
+ * - directed outputs.
*/
object IterateExample {
- private final val Bound = 100
+ final private val Bound = 100
def main(args: Array[String]): Unit = {
// Checking input parameters
@@ -63,18 +60,19 @@ object IterateExample {
// create input stream of integer pairs
val inputStream: DataStream[(Int, Int)] =
- if (params.has("input")) {
- // map a list of strings to integer pairs
- env.readTextFile(params.get("input")).map { value: String =>
- val record = value.substring(1, value.length - 1)
- val splitted = record.split(",")
- (Integer.parseInt(splitted(0)), Integer.parseInt(splitted(1)))
+ if (params.has("input")) {
+ // map a list of strings to integer pairs
+ env.readTextFile(params.get("input")).map {
+ value: String =>
+ val record = value.substring(1, value.length - 1)
+ val splitted = record.split(",")
+ (Integer.parseInt(splitted(0)), Integer.parseInt(splitted(1)))
+ }
+ } else {
+ println("Executing Iterate example with default input data set.")
+ println("Use --input to specify file input.")
+ env.addSource(new RandomFibonacciSource)
}
- } else {
- println("Executing Iterate example with default input data set.")
- println("Use --input to specify file input.")
- env.addSource(new RandomFibonacciSource)
- }
def withinBound(value: (Int, Int)) = value._1 < Bound && value._2 < Bound
@@ -87,8 +85,8 @@ object IterateExample {
.iterate(
(iteration: DataStream[(Int, Int, Int, Int, Int)]) => {
// calculates the next Fibonacci number and increment the counter
- val step = iteration.map(value =>
- (value._1, value._2, value._4, value._3 + value._4, value._5 + 1))
+ val step = iteration.map(
+ value => (value._1, value._2, value._4, value._3 + value._4, value._5 + 1))
// testing which tuple needs to be iterated again
val feedback = step.filter(value => withinBound(value._3, value._4))
// giving back the input pair and the counter
@@ -98,18 +96,24 @@ object IterateExample {
(feedback, output)
}
// timeout after 5 seconds
- , 5000L
+ ,
+ 5000L
)
if (params.has("output")) {
- numbers.sinkTo(FileSink.forRowFormat[((Int, Int), Int)](
- new Path(params.get("output")),
- new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ numbers
+ .sinkTo(
+ FileSink
+ .forRowFormat[((Int, Int), Int)](
+ new Path(params.get("output")),
+ new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
} else {
println("Printing result to stdout. Use --output to specify output path.")
@@ -123,9 +127,7 @@ object IterateExample {
// USER FUNCTIONS
// *************************************************************************
- /**
- * Generate BOUND number of random integer pairs from the range from 0 to BOUND/2
- */
+ /** Generate BOUND number of random integer pairs from the range from 0 to BOUND/2 */
private class RandomFibonacciSource extends SourceFunction[(Int, Int)] {
val rnd = new Random()
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
index 607ca04622f..d46f705ac1e 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.join
import org.apache.flink.api.java.utils.ParameterTool
@@ -26,11 +25,11 @@ import org.apache.flink.streaming.api.windowing.time.Time
/**
* Example illustrating a windowed stream join between two data streams.
*
- * The example works on two input streams with pairs (name, grade) and (name, salary)
- * respectively. It joins the streams based on "name" within a configurable window.
+ * The example works on two input streams with pairs (name, grade) and (name, salary) respectively.
+ * It joins the streams based on "name" within a configurable window.
*
- * The example uses a built-in sample data generator that generates
- * the streams of pairs at a configurable rate.
+ * The example uses a built-in sample data generator that generates the streams of pairs at a
+ * configurable rate.
*/
object WindowJoin {
@@ -55,8 +54,9 @@ object WindowJoin {
val rate = params.getLong("rate", 3)
println("Using windowSize=" + windowSize + ", data rate=" + rate)
- println("To customize example, use: WindowJoin " +
- "[--windowSize <window-size-in-millis>] [--rate <elements-per-second>]")
+ println(
+ "To customize example, use: WindowJoin " +
+ "[--windowSize <window-size-in-millis>] [--rate <elements-per-second>]")
// obtain execution environment, run this example in "ingestion time"
val env = StreamExecutionEnvironment.getExecutionEnvironment
@@ -79,16 +79,16 @@ object WindowJoin {
env.execute("Windowed Join Example")
}
-
def joinStreams(
grades: DataStream[Grade],
salaries: DataStream[Salary],
- windowSize: Long) : DataStream[Person] = {
+ windowSize: Long): DataStream[Person] = {
- grades.join(salaries)
+ grades
+ .join(salaries)
.where(_.name)
.equalTo(_.name)
.window(TumblingEventTimeWindows.of(Time.milliseconds(windowSize)))
- .apply { (g, s) => Person(g.name, g.grade, s.salary) }
+ .apply((g, s) => Person(g.name, g.grade, s.salary))
}
}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala
index 10c30c57b38..5a0e59ecd24 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala
@@ -15,45 +15,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.join
-import java.io.Serializable
-import java.util.Random
-
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.examples.utils.ThrottledIterator
import org.apache.flink.streaming.scala.examples.join.WindowJoin.{Grade, Salary}
+import java.io.Serializable
+import java.util.Random
+
import scala.collection.JavaConverters._
-/**
- * Sample data for the [[WindowJoin]] example.
- */
+/** Sample data for the [[WindowJoin]] example. */
object WindowJoinSampleData {
-
+
private[join] val NAMES = Array("tom", "jerry", "alice", "bob", "john", "grace")
private[join] val GRADE_COUNT = 5
private[join] val SALARY_MAX = 10000
- /**
- * Continuously generates (name, grade).
- */
+ /** Continuously generates (name, grade). */
def getGradeSource(env: StreamExecutionEnvironment, rate: Long): DataStream[Grade] = {
- env.fromCollection(new ThrottledIterator(new GradeSource().asJava, rate).asScala)
+ env.fromCollection(new ThrottledIterator(new GradeSource().asJava, rate).asScala)
}
- /**
- * Continuously generates (name, salary).
- */
+ /** Continuously generates (name, salary). */
def getSalarySource(env: StreamExecutionEnvironment, rate: Long): DataStream[Salary] = {
env.fromCollection(new ThrottledIterator(new SalarySource().asJava, rate).asScala)
}
-
+
// --------------------------------------------------------------------------
-
+
class GradeSource extends Iterator[Grade] with Serializable {
-
+
private[this] val rnd = new Random(hashCode())
def hasNext: Boolean = true
@@ -62,7 +55,7 @@ object WindowJoinSampleData {
Grade(NAMES(rnd.nextInt(NAMES.length)), rnd.nextInt(GRADE_COUNT) + 1)
}
}
-
+
class SalarySource extends Iterator[Salary] with Serializable {
private[this] val rnd = new Random(hashCode())
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala
index fbb98ede71d..63b7d2478ca 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.socket
import org.apache.flink.api.java.utils.ParameterTool
@@ -25,10 +24,9 @@ import org.apache.flink.streaming.api.windowing.time.Time
/**
* Implements a streaming windowed version of the "WordCount" program.
- *
- * This program connects to a server socket and reads strings from the socket.
- * The easiest way to try this out is to open a text sever (at port 12345)
- * using the ''netcat'' tool via
+ *
+ * This program connects to a server socket and reads strings from the socket. The easiest way to
+ * try this out is to open a text sever (at port 12345) using the ''netcat'' tool via
* {{{
* nc -l 12345 on Linux or nc -l -p 12345 on Windows
* }}}
@@ -37,7 +35,7 @@ import org.apache.flink.streaming.api.windowing.time.Time
object SocketWindowWordCount {
/** Main program method */
- def main(args: Array[String]) : Unit = {
+ def main(args: Array[String]): Unit = {
// the host and the port to connect to
var hostname: String = "localhost"
@@ -52,25 +50,26 @@ object SocketWindowWordCount {
System.err.println("No port specified. Please run 'SocketWindowWordCount " +
"--hostname <hostname> --port <port>', where hostname (localhost by default) and port " +
"is the address of the text server")
- System.err.println("To start a simple text server, run 'netcat -l <port>' " +
- "and type the input text into the command line")
+ System.err.println(
+ "To start a simple text server, run 'netcat -l <port>' " +
+ "and type the input text into the command line")
return
}
}
-
+
// get the execution environment
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
-
+
// get input data by connecting to the socket
val text: DataStream[String] = env.socketTextStream(hostname, port, '\n')
- // parse the data, group it, window it, and aggregate the counts
+ // parse the data, group it, window it, and aggregate the counts
val windowCounts = text
- .flatMap { w => w.split("\\s") }
- .map { w => WordWithCount(w, 1) }
- .keyBy(_.word)
- .window(TumblingProcessingTimeWindows.of(Time.seconds(5)))
- .sum("count")
+ .flatMap(w => w.split("\\s"))
+ .map(w => WordWithCount(w, 1))
+ .keyBy(_.word)
+ .window(TumblingProcessingTimeWindows.of(Time.seconds(5)))
+ .sum("count")
// print the results with a single thread, rather than in parallel
windowCounts.print()
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala
index a982feb50b2..49e0f76dd54 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
import org.apache.flink.api.scala._
@@ -27,8 +26,8 @@ import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeW
import org.apache.flink.streaming.api.windowing.time.Time
/**
- * An example of grouped stream windowing into sliding time windows.
- * This example uses [[RichParallelSourceFunction]] to generate a list of key-value pair.
+ * An example of grouped stream windowing into sliding time windows. This example uses
+ * [[RichParallelSourceFunction]] to generate a list of key-value pair.
*/
object GroupedProcessingTimeWindowExample {
@@ -47,9 +46,7 @@ object GroupedProcessingTimeWindowExample {
env.execute()
}
- /**
- * Parallel data source that serves a list of key-value pair.
- */
+ /** Parallel data source that serves a list of key-value pair. */
private class DataSource extends RichParallelSourceFunction[(Long, Long)] {
@volatile private var running = true
@@ -74,7 +71,7 @@ object GroupedProcessingTimeWindowExample {
}
val endTime = System.currentTimeMillis()
- println(s"Took ${endTime - startTime} msecs for ${numElements} values")
+ println(s"Took ${endTime - startTime} msecs for $numElements values")
}
override def cancel(): Unit = running = false
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala
index a5d0b7e497f..c537f316d66 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
import org.apache.flink.api.common.serialization.SimpleStringEncoder
@@ -35,8 +34,8 @@ import org.apache.flink.streaming.api.windowing.time.Time
import java.time.Duration
/**
- * An example of grouped stream windowing in session windows with session timeout of 3 msec.
- * A source fetches elements with key, timestamp, and count.
+ * An example of grouped stream windowing in session windows with session timeout of 3 msec. A
+ * source fetches elements with key, timestamp, and count.
*/
object SessionWindowing {
@@ -62,14 +61,15 @@ object SessionWindowing {
("c", 11L, 1)
)
- val source: DataStream[(String, Long, Int)] = env.addSource(
- new SourceFunction[(String, Long, Int)]() {
+ val source: DataStream[(String, Long, Int)] =
+ env.addSource(new SourceFunction[(String, Long, Int)]() {
override def run(ctx: SourceContext[(String, Long, Int)]): Unit = {
- input.foreach(value => {
- ctx.collectWithTimestamp(value, value._2)
- ctx.emitWatermark(new Watermark(value._2 - 1))
- })
+ input.foreach(
+ value => {
+ ctx.collectWithTimestamp(value, value._2)
+ ctx.emitWatermark(new Watermark(value._2 - 1))
+ })
ctx.emitWatermark(new Watermark(Long.MaxValue))
}
@@ -84,14 +84,19 @@ object SessionWindowing {
.sum(2)
if (fileOutput) {
- aggregated.sinkTo(FileSink.forRowFormat[(String, Long, Int)](
- new Path(params.get("output")),
- new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ aggregated
+ .sinkTo(
+ FileSink
+ .forRowFormat[(String, Long, Int)](
+ new Path(params.get("output")),
+ new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
} else {
print("Printing result to stdout. Use --output to specify output path.")
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index d487488cfcb..4a0d84e06cd 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -15,10 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
-
import org.apache.flink.api.common.eventtime.WatermarkStrategy
import org.apache.flink.api.common.serialization.SimpleStringEncoder
import org.apache.flink.api.java.utils.ParameterTool
@@ -42,12 +40,10 @@ import java.time.Duration
import java.util.concurrent.TimeUnit
/**
- * An example of grouped stream windowing where different eviction and
- * trigger policies can be used. A source fetches events from cars
- * every 100 msec containing their id, their current speed (kmh),
- * overall elapsed distance (m) and a timestamp. The streaming
- * example triggers the top speed of each car every x meters elapsed
- * for the last y seconds.
+ * An example of grouped stream windowing where different eviction and trigger policies can be used.
+ * A source fetches events from cars every 100 msec containing their id, their current speed (kmh),
+ * overall elapsed distance (m) and a timestamp. The streaming example triggers the top speed of
+ * each car every x meters elapsed for the last y seconds.
*/
object TopSpeedWindowing {
@@ -97,11 +93,12 @@ object TopSpeedWindowing {
case Some(input) =>
// Create a new file source that will read files from a given set of directories.
// Each file will be processed as plain text and split based on newlines.
- val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input:_*)
- params.discoveryInterval.foreach { duration =>
- // If a discovery interval is provided, the source will
- // continuously watch the given directories for new files.
- builder.monitorContinuously(duration)
+ val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input: _*)
+ params.discoveryInterval.foreach {
+ duration =>
+ // If a discovery interval is provided, the source will
+ // continuously watch the given directories for new files.
+ builder.monitorContinuously(duration)
}
env
.fromSource(builder.build(), WatermarkStrategy.noWatermarks(), "file-input")
@@ -112,13 +109,17 @@ object TopSpeedWindowing {
}
val topSpeeds = cars
- .assignAscendingTimestamps( _.time )
+ .assignAscendingTimestamps(_.time)
.keyBy(_.carId)
.window(GlobalWindows.create)
.evictor(TimeEvictor.of(Time.of(evictionSec * 1000, TimeUnit.MILLISECONDS)))
- .trigger(DeltaTrigger.of(triggerMeters, new DeltaFunction[CarEvent] {
- def getDelta(oldSp: CarEvent, newSp: CarEvent): Double = newSp.distance - oldSp.distance
- }, cars.dataType.createSerializer(env.getConfig)))
+ .trigger(DeltaTrigger.of(
+ triggerMeters,
+ new DeltaFunction[CarEvent] {
+ def getDelta(oldSp: CarEvent, newSp: CarEvent): Double = newSp.distance - oldSp.distance
+ },
+ cars.dataType.createSerializer(env.getConfig)
+ ))
// .window(Time.of(evictionSec * 1000, (car : CarEvent) => car.time))
// .every(Delta.of[CarEvent](triggerMeters,
// (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0)))
@@ -129,12 +130,17 @@ object TopSpeedWindowing {
// Given an output directory, Flink will write the results to a file
// using a simple string encoding. In a production environment, this might
// be something more structured like CSV, Avro, JSON, or Parquet.
- topSpeeds.sinkTo(FileSink.forRowFormat[CarEvent](output, new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ topSpeeds
+ .sinkTo(
+ FileSink
+ .forRowFormat[CarEvent](output, new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
case None => topSpeeds.print().name("print-sink")
@@ -148,7 +154,7 @@ object TopSpeedWindowing {
// USER FUNCTIONS
// *************************************************************************
- def parseMap(line : String): CarEvent = {
+ def parseMap(line: String): CarEvent = {
val record = line.substring(1, line.length - 1).split(",")
CarEvent(record(0).toInt, record(1).toInt, record(2).toDouble, record(3).toLong)
}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala
index 6ba8810a194..ab1e1ce4a1e 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
import org.apache.flink.api.common.eventtime.WatermarkStrategy
@@ -47,15 +46,13 @@ import java.time.Duration
* --slide <n>
* }}}
*
- * If no parameters are provided, the program is run with default data from
- * [[WordCountData]].
+ * If no parameters are provided, the program is run with default data from [[WordCountData]].
*
* This example shows how to:
*
- * - write a simple Flink Streaming program,
- * - use tuple data types,
- * - use basic windowing abstractions.
- *
+ * - write a simple Flink Streaming program,
+ * - use tuple data types,
+ * - use basic windowing abstractions.
*/
object WindowWordCount {
@@ -95,15 +92,16 @@ object WindowWordCount {
case Some(input) =>
// Create a new file source that will read files from a given set of directories.
// Each file will be processed as plain text and split based on newlines.
- val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input:_*)
- params.discoveryInterval.foreach { duration =>
- // If a discovery interval is provided, the source will
- // continuously watch the given directories for new files.
- builder.monitorContinuously(duration)
+ val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input: _*)
+ params.discoveryInterval.foreach {
+ duration =>
+ // If a discovery interval is provided, the source will
+ // continuously watch the given directories for new files.
+ builder.monitorContinuously(duration)
}
env.fromSource(builder.build(), WatermarkStrategy.noWatermarks(), "file-input")
case None =>
- env.fromElements(WordCountData.WORDS:_*).name("in-memory-input")
+ env.fromElements(WordCountData.WORDS: _*).name("in-memory-input")
}
val windowSize = params.getInt("window").getOrElse(250)
@@ -113,7 +111,8 @@ object WindowWordCount {
// The text lines read from the source are split into words
// using a user-defined function. The tokenizer, implemented below,
// will output each words as a (2-tuple) containing (word, 1)
- text.flatMap(new Tokenizer)
+ text
+ .flatMap(new Tokenizer)
.name("tokenizer")
// keyBy groups tuples based on the "_1" field, the word.
// Using a keyBy allows performing aggregations and other
@@ -134,12 +133,17 @@ object WindowWordCount {
// Given an output directory, Flink will write the results to a file
// using a simple string encoding. In a production environment, this might
// be something more structured like CSV, Avro, JSON, or Parquet.
- counts.sinkTo(FileSink.forRowFormat[(String, Int)](output, new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ counts
+ .sinkTo(
+ FileSink
+ .forRowFormat[(String, Int)](output, new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
case None => counts.print().name("print-sink")
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala
index 8f1d4c4d3a5..d216f249f5d 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala
@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing.util
+import org.apache.flink.api.java.tuple.{Tuple4 => JTuple4}
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.examples.windowing.util.{CarSource => JCarSource}
import org.apache.flink.streaming.scala.examples.windowing.TopSpeedWindowing.CarEvent
-import org.apache.flink.api.java.tuple.{Tuple4 => JTuple4}
-import java.lang.{Integer => JInt, Double => JDouble, Long => JLong}
+import java.lang.{Double => JDouble, Integer => JInt, Long => JLong}
/** A simple in-memory source. */
object CarSource {
@@ -48,8 +47,8 @@ private class WrappingCollector(ctx: SourceFunction.SourceContext[CarEvent])
ctx.collect(CarEvent(element.f0, element.f1, element.f2, element.f3))
override def collectWithTimestamp(
- element: JTuple4[JInt, JInt, JDouble, JLong],
- timestamp: Long): Unit =
+ element: JTuple4[JInt, JInt, JDouble, JLong],
+ timestamp: Long): Unit =
ctx.collectWithTimestamp(CarEvent(element.f0, element.f1, element.f2, element.f3), timestamp)
override def emitWatermark(mark: Watermark): Unit = ctx.emitWatermark(mark)
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala
index 26004028952..c19aa3ff2dc 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.wordcount
import org.apache.flink.api.common.eventtime.WatermarkStrategy
@@ -41,25 +40,23 @@ import java.time.Duration
*
* Usage:
*
- * {{{ --input <path> }}} A list of input files and / or directories to read.
- * If no input is provided, the program is run with default data from [[WordCountData]].
+ * {{{--input <path>}}} A list of input files and / or directories to read. If no input is provided,
+ * the program is run with default data from [[WordCountData]].
*
- * {{{--discovery-interval <duration> }}} Turns the file reader
- * into a continuous source that will monitor the provided input directories
- * every interval and read any new files.
+ * {{{--discovery-interval <duration>}}} Turns the file reader into a continuous source that will
+ * monitor the provided input directories every interval and read any new files.
*
- * {{{--output <path> }}} The output directory where the Job will
- * write the results. If no output path is provided, the Job will print the results
- * to `stdout`
+ * {{{--output <path>}}} The output directory where the Job will write the results. If no output
+ * path is provided, the Job will print the results to `stdout`
*
- * {{{--execution-mode <mode> }}} The execution mode (BATCH, STREAMING, or AUTOMATIC) of this
+ * {{{--execution-mode <mode>}}} The execution mode (BATCH, STREAMING, or AUTOMATIC) of this
* pipeline.
*
* This example shows how to:
*
- * - Write a simple Flink DataStream program
- * - Use tuple data types
- * - Write and use a user-defined function
+ * - Write a simple Flink DataStream program
+ * - Use tuple data types
+ * - Write and use a user-defined function
*/
object WordCount {
@@ -104,22 +101,24 @@ object WordCount {
case Some(input) =>
// Create a new file source that will read files from a given set of directories.
// Each file will be processed as plain text and split based on newlines.
- val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input:_*)
- params.discoveryInterval.foreach { duration =>
- // If a discovery interval is provided, the source will
- // continuously watch the given directories for new files.
- builder.monitorContinuously(duration)
+ val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input: _*)
+ params.discoveryInterval.foreach {
+ duration =>
+ // If a discovery interval is provided, the source will
+ // continuously watch the given directories for new files.
+ builder.monitorContinuously(duration)
}
env.fromSource(builder.build(), WatermarkStrategy.noWatermarks(), "file-input")
case None =>
- env.fromElements(WordCountData.WORDS:_*).name("in-memory-input")
+ env.fromElements(WordCountData.WORDS: _*).name("in-memory-input")
}
val counts =
// The text lines read from the source are split into words
// using a user-defined function. The tokenizer, implemented below,
// will output each word as a (2-tuple) containing (word, 1)
- text.flatMap(new Tokenizer)
+ text
+ .flatMap(new Tokenizer)
.name("tokenizer")
// keyBy groups tuples based on the "_1" field, the word.
// Using a keyBy allows performing aggregations and other
@@ -138,12 +137,17 @@ object WordCount {
// Given an output directory, Flink will write the results to a file
// using a simple string encoding. In a production environment, this might
// be something more structured like CSV, Avro, JSON, or Parquet.
- counts.sinkTo(FileSink.forRowFormat[(String, Int)](output, new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ counts
+ .sinkTo(
+ FileSink
+ .forRowFormat[(String, Int)](output, new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
case None => counts.print().name("print-sink")
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala
index f7f7b0b8d7d..cff44ba2897 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.wordcount.util
import org.apache.flink.api.common.{ExecutionConfig, RuntimeExecutionMode}
diff --git a/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala b/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
index d4fcb5e840f..16fce22fb36 100644
--- a/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
+++ b/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
@@ -15,12 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples
-import java.io.File
-
-import org.apache.commons.io.FileUtils
import org.apache.flink.core.fs.FileSystem.WriteMode
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala._
@@ -33,11 +29,13 @@ import org.apache.flink.streaming.scala.examples.wordcount.WordCount
import org.apache.flink.streaming.test.examples.join.WindowJoinData
import org.apache.flink.test.testdata.WordCountData
import org.apache.flink.test.util.{AbstractTestBase, TestBaseUtils}
+
+import org.apache.commons.io.FileUtils
import org.junit.Test
-/**
- * Integration test for streaming programs in Scala examples.
- */
+import java.io.File
+
+/** Integration test for streaming programs in Scala examples. */
class StreamingExamplesITCase extends AbstractTestBase {
@Test
@@ -48,10 +46,13 @@ class StreamingExamplesITCase extends AbstractTestBase {
// the example is inherently non-deterministic. The iteration timeout of 5000 ms
// is frequently not enough to make the test run stable on CI infrastructure
// with very small containers, so we cannot do a validation here
- IterateExample.main(Array(
- "--input", inputPath,
- "--output", resultPath
- ))
+ IterateExample.main(
+ Array(
+ "--input",
+ inputPath,
+ "--output",
+ resultPath
+ ))
}
@Test
@@ -63,31 +64,34 @@ class StreamingExamplesITCase extends AbstractTestBase {
val grades = env
.fromCollection(WindowJoinData.GRADES_INPUT.split("\n"))
- .map( line => {
- val fields = line.split(",")
- Grade(fields(1), fields(2).toInt)
- })
+ .map(
+ line => {
+ val fields = line.split(",")
+ Grade(fields(1), fields(2).toInt)
+ })
val salaries = env
.fromCollection(WindowJoinData.SALARIES_INPUT.split("\n"))
- .map( line => {
- val fields = line.split(",")
- Salary(fields(1), fields(2).toInt)
- })
-
- WindowJoin.joinStreams(grades, salaries, 100)
+ .map(
+ line => {
+ val fields = line.split(",")
+ Salary(fields(1), fields(2).toInt)
+ })
+
+ WindowJoin
+ .joinStreams(grades, salaries, 100)
.writeAsText(resultPath, WriteMode.OVERWRITE)
env.execute()
TestBaseUtils.checkLinesAgainstRegexp(resultPath, "^Person\\([a-z]+,(\\d),(\\d)+\\)")
- }
- finally try
- FileUtils.deleteDirectory(new File(resultPath))
+ } finally
+ try
+ FileUtils.deleteDirectory(new File(resultPath))
- catch {
- case _: Throwable =>
- }
+ catch {
+ case _: Throwable =>
+ }
}
@Test
@@ -103,13 +107,19 @@ class StreamingExamplesITCase extends AbstractTestBase {
val textPath = createTempFile("text.txt", WordCountData.TEXT)
val resultPath = getTempDirPath("result")
- WindowWordCount.main(Array(
- "--input", textPath,
- "--output", resultPath,
- "--window", windowSize,
- "--slide", slideSize,
- "--execution-mode", "AUTOMATIC"
- ))
+ WindowWordCount.main(
+ Array(
+ "--input",
+ textPath,
+ "--output",
+ resultPath,
+ "--window",
+ windowSize,
+ "--slide",
+ slideSize,
+ "--execution-mode",
+ "AUTOMATIC"
+ ))
// since the parallel tokenizers might have different speed
// the exact output can not be checked just whether it is well-formed
@@ -122,14 +132,16 @@ class StreamingExamplesITCase extends AbstractTestBase {
val textPath = createTempFile("text.txt", WordCountData.TEXT)
val resultPath = getTempDirPath("result")
- WordCount.main(Array(
- "--input", textPath,
- "--output", resultPath,
- "--execution-mode", "automatic"
- ))
-
- TestBaseUtils.compareResultsByLinesInMemory(
- WordCountData.COUNTS_AS_TUPLES,
- resultPath)
+ WordCount.main(
+ Array(
+ "--input",
+ textPath,
+ "--output",
+ resultPath,
+ "--execution-mode",
+ "automatic"
+ ))
+
+ TestBaseUtils.compareResultsByLinesInMemory(WordCountData.COUNTS_AS_TUPLES, resultPath)
}
}
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala
index 11c81b90792..2b18b27be3a 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.table.examples.scala.basics
import org.apache.flink.table.api.{EnvironmentSettings, TableEnvironment, _}
@@ -31,12 +30,13 @@ import scala.collection.JavaConverters._
* Example for getting started with the Table & SQL API in Scala.
*
* The example shows how to create, transform, and query a table. It should give a first impression
- * about the look-and-feel of the API without going too much into details. See the other examples for
- * using connectors or more complex operations.
+ * about the look-and-feel of the API without going too much into details. See the other examples
+ * for using connectors or more complex operations.
*
* In particular, the example shows how to
* - setup a [[TableEnvironment]],
- * - use the environment for creating example tables, registering views, and executing SQL queries,
+ * - use the environment for creating example tables, registering views, and executing SQL
+ * queries,
* - transform tables with filters and projections,
* - declare user-defined functions,
* - and print/collect results locally.
@@ -49,19 +49,80 @@ object GettingStartedExample {
// setup the unified API
// in this case: declare that the table programs should be executed in batch mode
- val settings = EnvironmentSettings.newInstance()
+ val settings = EnvironmentSettings
+ .newInstance()
.inBatchMode()
.build()
val env = TableEnvironment.create(settings)
// create a table with example data without a connector required
val rawCustomers = env.fromValues(
- row("Guillermo Smith", LocalDate.parse("1992-12-12"), "4081 Valley Road", "08540", "New Jersey", "m", true, 0, 78, 3),
- row("Valeria Mendoza", LocalDate.parse("1970-03-28"), "1239 Rainbow Road", "90017", "Los Angeles", "f", true, 9, 39, 0),
- row("Leann Holloway", LocalDate.parse("1989-05-21"), "2359 New Street", "97401", "Eugene", null, true, null, null, null),
- row("Brandy Sanders", LocalDate.parse("1956-05-26"), "4891 Walkers-Ridge-Way", "73119", "Oklahoma City", "m", false, 9, 39, 0),
- row("John Turner", LocalDate.parse("1982-10-02"), "2359 New Street", "60605", "Chicago", "m", true, 12, 39, 0),
- row("Ellen Ortega", LocalDate.parse("1985-06-18"), "2448 Rodney STreet", "85023", "Phoenix", "f", true, 0, 78, 3)
+ row(
+ "Guillermo Smith",
+ LocalDate.parse("1992-12-12"),
+ "4081 Valley Road",
+ "08540",
+ "New Jersey",
+ "m",
+ true,
+ 0,
+ 78,
+ 3),
+ row(
+ "Valeria Mendoza",
+ LocalDate.parse("1970-03-28"),
+ "1239 Rainbow Road",
+ "90017",
+ "Los Angeles",
+ "f",
+ true,
+ 9,
+ 39,
+ 0),
+ row(
+ "Leann Holloway",
+ LocalDate.parse("1989-05-21"),
+ "2359 New Street",
+ "97401",
+ "Eugene",
+ null,
+ true,
+ null,
+ null,
+ null),
+ row(
+ "Brandy Sanders",
+ LocalDate.parse("1956-05-26"),
+ "4891 Walkers-Ridge-Way",
+ "73119",
+ "Oklahoma City",
+ "m",
+ false,
+ 9,
+ 39,
+ 0),
+ row(
+ "John Turner",
+ LocalDate.parse("1982-10-02"),
+ "2359 New Street",
+ "60605",
+ "Chicago",
+ "m",
+ true,
+ 12,
+ 39,
+ 0),
+ row(
+ "Ellen Ortega",
+ LocalDate.parse("1985-06-18"),
+ "2448 Rodney STreet",
+ "85023",
+ "Phoenix",
+ "f",
+ true,
+ 0,
+ 78,
+ 3)
)
// handle ranges of columns easily
@@ -78,12 +139,11 @@ object GettingStartedExample {
// call execute() and print() to get insights
env
.sqlQuery("""
- |SELECT
- | COUNT(*) AS `number of customers`,
- | AVG(YEAR(date_of_birth)) AS `average birth year`
- |FROM `customers`
- |""".stripMargin
- )
+ |SELECT
+ | COUNT(*) AS `number of customers`,
+ | AVG(YEAR(date_of_birth)) AS `average birth year`
+ |FROM `customers`
+ |""".stripMargin)
.execute()
.print()
@@ -108,7 +168,10 @@ object GettingStartedExample {
val actualOutput = iterator.asScala.toSet
val expectedOutput = Set(
- Row.of("GUILLERMO SMITH", LocalDate.parse("1992-12-12"), "4081 VALLEY ROAD, 08540, NEW JERSEY"),
+ Row.of(
+ "GUILLERMO SMITH",
+ LocalDate.parse("1992-12-12"),
+ "4081 VALLEY ROAD, 08540, NEW JERSEY"),
Row.of("JOHN TURNER", LocalDate.parse("1982-10-02"), "2359 NEW STREET, 60605, CHICAGO"),
Row.of("ELLEN ORTEGA", LocalDate.parse("1985-06-18"), "2448 RODNEY STREET, 85023, PHOENIX")
)
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala
index 3eb02119a63..6bfb061b0f3 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala
@@ -47,15 +47,11 @@ object StreamSQLExample {
// set up the Scala Table API
val tableEnv = StreamTableEnvironment.create(env)
- val orderA = env.fromCollection(Seq(
- Order(1L, "beer", 3),
- Order(1L, "diaper", 4),
- Order(3L, "rubber", 2)))
+ val orderA =
+ env.fromCollection(Seq(Order(1L, "beer", 3), Order(1L, "diaper", 4), Order(3L, "rubber", 2)))
- val orderB = env.fromCollection(Seq(
- Order(2L, "pen", 3),
- Order(2L, "rubber", 3),
- Order(4L, "beer", 1)))
+ val orderB =
+ env.fromCollection(Seq(Order(2L, "pen", 3), Order(2L, "rubber", 3), Order(4L, "beer", 1)))
// convert the first DataStream to a Table object
// it will be used "inline" and is not registered in a catalog
@@ -66,11 +62,10 @@ object StreamSQLExample {
tableEnv.createTemporaryView("TableB", orderB)
// union the two tables
- val result = tableEnv.sqlQuery(
- s"""
- |SELECT * FROM $tableA WHERE amount > 2
- |UNION ALL
- |SELECT * FROM TableB WHERE amount < 2
+ val result = tableEnv.sqlQuery(s"""
+ |SELECT * FROM $tableA WHERE amount > 2
+ |UNION ALL
+ |SELECT * FROM TableB WHERE amount < 2
""".stripMargin)
// convert the Table back to an insert-only DataStream of type `Order`
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala
index bf87231c38f..1e71f8e52af 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala
@@ -23,12 +23,12 @@ import org.apache.flink.table.api._
import org.apache.flink.table.api.bridge.scala._
/**
- * Simple example for demonstrating the use of Table API on a Stream Table.
- *
- * This example shows how to:
- * - Convert DataStreams to Tables
- * - Apply union, select, and filter operations
- */
+ * Simple example for demonstrating the use of Table API on a Stream Table.
+ *
+ * This example shows how to:
+ * - Convert DataStreams to Tables
+ * - Apply union, select, and filter operations
+ */
object StreamTableExample {
// *************************************************************************
@@ -41,18 +41,17 @@ object StreamTableExample {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tEnv = StreamTableEnvironment.create(env)
- val orderA = env.fromCollection(Seq(
- Order(1L, "beer", 3),
- Order(1L, "diaper", 4),
- Order(3L, "rubber", 2))).toTable(tEnv)
+ val orderA = env
+ .fromCollection(Seq(Order(1L, "beer", 3), Order(1L, "diaper", 4), Order(3L, "rubber", 2)))
+ .toTable(tEnv)
- val orderB = env.fromCollection(Seq(
- Order(2L, "pen", 3),
- Order(2L, "rubber", 3),
- Order(4L, "beer", 1))).toTable(tEnv)
+ val orderB = env
+ .fromCollection(Seq(Order(2L, "pen", 3), Order(2L, "rubber", 3), Order(4L, "beer", 1)))
+ .toTable(tEnv)
// union the two tables
- val result: DataStream[Order] = orderA.unionAll(orderB)
+ val result: DataStream[Order] = orderA
+ .unionAll(orderB)
.select('user, 'product, 'amount)
.where('amount > 2)
.toDataStream(classOf[Order])
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala
index 7d220a6e6f4..42a772c66c9 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.table.examples.scala.basics
import org.apache.flink.table.api._
@@ -26,23 +25,25 @@ object WordCountSQLExample {
def main(args: Array[String]): Unit = {
// set up the Table API
- val settings = EnvironmentSettings.newInstance()
+ val settings = EnvironmentSettings
+ .newInstance()
.inBatchMode()
.build()
val tableEnv = TableEnvironment.create(settings)
// execute a Flink SQL job and print the result locally
- tableEnv.executeSql(
- // define the aggregation
- "SELECT word, SUM(frequency) AS `count`\n"
+ tableEnv
+ .executeSql(
+ // define the aggregation
+ "SELECT word, SUM(frequency) AS `count`\n"
// read from an artificial fixed-size table with rows and columns
- + "FROM (\n"
- + " VALUES ('Hello', 1), ('Ciao', 1), ('Hello', 2)\n"
- + ")\n"
- // name the table and its columns
- + "AS WordTable(word, frequency)\n"
- // group for aggregation
- + "GROUP BY word")
+ + "FROM (\n"
+ + " VALUES ('Hello', 1), ('Ciao', 1), ('Hello', 2)\n"
+ + ")\n"
+ // name the table and its columns
+ + "AS WordTable(word, frequency)\n"
+ // group for aggregation
+ + "GROUP BY word")
.print()
}
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala
index 565da3178fe..53edb6a2323 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala
@@ -17,45 +17,50 @@
*/
package org.apache.flink.cep.scala
+import org.apache.flink.cep.{CEP => JCEP, EventComparator}
import org.apache.flink.cep.scala.pattern.Pattern
-import org.apache.flink.cep.{EventComparator, CEP => JCEP}
import org.apache.flink.streaming.api.scala.DataStream
-/**
- * Utility method to transform a [[DataStream]] into a [[PatternStream]] to do CEP.
- */
+/** Utility method to transform a [[DataStream]] into a [[PatternStream]] to do CEP. */
object CEP {
+
/**
- * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API.
- * See [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying
- * Java API works.
- *
- * @param input DataStream containing the input events
- * @param pattern Pattern specification which shall be detected
- * @tparam T Type of the input events
- * @return Resulting pattern stream
- */
+ * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API. See
+ * [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying Java API works.
+ *
+ * @param input
+ * DataStream containing the input events
+ * @param pattern
+ * Pattern specification which shall be detected
+ * @tparam T
+ * Type of the input events
+ * @return
+ * Resulting pattern stream
+ */
def pattern[T](input: DataStream[T], pattern: Pattern[T, _ <: T]): PatternStream[T] = {
wrapPatternStream(JCEP.pattern(input.javaStream, pattern.wrappedPattern))
}
/**
- * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API.
- * See [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying
- * Java API works.
- *
- * @param input DataStream containing the input events
- * @param pattern Pattern specification which shall be detected
- * @param comparator Comparator to sort events with equal timestamps
- * @tparam T Type of the input events
- * @return Resulting pattern stream
- */
+ * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API. See
+ * [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying Java API works.
+ *
+ * @param input
+ * DataStream containing the input events
+ * @param pattern
+ * Pattern specification which shall be detected
+ * @param comparator
+ * Comparator to sort events with equal timestamps
+ * @tparam T
+ * Type of the input events
+ * @return
+ * Resulting pattern stream
+ */
def pattern[T](
- input: DataStream[T],
- pattern: Pattern[T, _ <: T],
- comparator: EventComparator[T]): PatternStream[T] = {
+ input: DataStream[T],
+ pattern: Pattern[T, _ <: T],
+ comparator: EventComparator[T]): PatternStream[T] = {
wrapPatternStream(JCEP.pattern(input.javaStream, pattern.wrappedPattern, comparator))
}
}
-
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
index ebc1c27c8ea..215a914d59a 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
@@ -17,168 +17,187 @@
*/
package org.apache.flink.cep.scala
-import java.util.{UUID, List => JList, Map => JMap}
-
import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.cep.{PatternFlatSelectFunction, PatternFlatTimeoutFunction, PatternSelectFunction, PatternStream => JPatternStream, PatternTimeoutFunction}
import org.apache.flink.cep.functions.PatternProcessFunction
-import org.apache.flink.cep.{PatternFlatSelectFunction, PatternFlatTimeoutFunction, PatternSelectFunction, PatternTimeoutFunction, PatternStream => JPatternStream}
import org.apache.flink.streaming.api.scala.{asScalaStream, _}
import org.apache.flink.util.Collector
+import java.util.{List => JList, Map => JMap, UUID}
+
import scala.collection.Map
/**
- * Stream abstraction for CEP pattern detection. A pattern stream is a stream which emits detected
- * pattern sequences as a map of events associated with their names. The pattern is detected using
- * a [[org.apache.flink.cep.nfa.NFA]]. In order to process the detected sequences, the user has to
- * specify a [[PatternSelectFunction]] or a [[PatternFlatSelectFunction]].
- *
- * @param jPatternStream Underlying pattern stream from Java API
- * @tparam T Type of the events
- */
+ * Stream abstraction for CEP pattern detection. A pattern stream is a stream which emits detected
+ * pattern sequences as a map of events associated with their names. The pattern is detected using a
+ * [[org.apache.flink.cep.nfa.NFA]]. In order to process the detected sequences, the user has to
+ * specify a [[PatternSelectFunction]] or a [[PatternFlatSelectFunction]].
+ *
+ * @param jPatternStream
+ * Underlying pattern stream from Java API
+ * @tparam T
+ * Type of the events
+ */
class PatternStream[T](jPatternStream: JPatternStream[T]) {
private[flink] def wrappedPatternStream = jPatternStream
/**
- * Applies a process function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternProcessFunction]] is called.
- *
- * @param patternProcessFunction The pattern process function which is called for each detected
- * pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern select function.
- */
- def process[R: TypeInformation](patternProcessFunction: PatternProcessFunction[T, R])
- : DataStream[R] = {
+ * Applies a process function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternProcessFunction]] is called.
+ *
+ * @param patternProcessFunction
+ * The pattern process function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern select function.
+ */
+ def process[R: TypeInformation](
+ patternProcessFunction: PatternProcessFunction[T, R]): DataStream[R] = {
asScalaStream(jPatternStream.process(patternProcessFunction, implicitly[TypeInformation[R]]))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern select function.
- */
- def select[R: TypeInformation](patternSelectFunction: PatternSelectFunction[T, R])
- : DataStream[R] = {
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern select function.
+ */
+ def select[R: TypeInformation](
+ patternSelectFunction: PatternSelectFunction[T, R]): DataStream[R] = {
asScalaStream(jPatternStream.select(patternSelectFunction, implicitly[TypeInformation[R]]))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting timeout event.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contains the resulting events and resulting timeout
- * events.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting timeout event.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contains the resulting events and resulting timeout events.
+ */
@deprecated
def select[L: TypeInformation, R: TypeInformation](
- patternTimeoutFunction: PatternTimeoutFunction[T, L],
- patternSelectFunction: PatternSelectFunction[T, R])
- : DataStream[Either[L, R]] = {
+ patternTimeoutFunction: PatternTimeoutFunction[T, L],
+ patternSelectFunction: PatternSelectFunction[T, R]): DataStream[Either[L, R]] = {
val outputTag = OutputTag[L](UUID.randomUUID().toString)
val mainStream = select(outputTag, patternTimeoutFunction, patternSelectFunction)
mainStream.connect(mainStream.getSideOutput[L](outputTag)).map(r => Right(r), l => Left(l))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting timeout event.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream which contains the resulting elements with the resulting timeout elements
- * in a side output.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting timeout event.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream which contains the resulting elements with the resulting timeout elements in a
+ * side output.
+ */
def select[L: TypeInformation, R: TypeInformation](
- outputTag: OutputTag[L],
- patternTimeoutFunction: PatternTimeoutFunction[T, L],
- patternSelectFunction: PatternSelectFunction[T, R])
- : DataStream[R] = {
+ outputTag: OutputTag[L],
+ patternTimeoutFunction: PatternTimeoutFunction[T, L],
+ patternSelectFunction: PatternSelectFunction[T, R]): DataStream[R] = {
val cleanedSelect = cleanClosure(patternSelectFunction)
val cleanedTimeout = cleanClosure(patternTimeoutFunction)
asScalaStream(
jPatternStream
- .select(outputTag, cleanedTimeout, implicitly[TypeInformation[R]], cleanedSelect)
+ .select(outputTag, cleanedTimeout, implicitly[TypeInformation[R]], cleanedSelect)
)
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern flat select
- * function.
- */
- def flatSelect[R: TypeInformation](patternFlatSelectFunction: PatternFlatSelectFunction[T, R])
- : DataStream[R] = {
- asScalaStream(jPatternStream
- .flatSelect(patternFlatSelectFunction, implicitly[TypeInformation[R]]))
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern flat select function.
+ */
+ def flatSelect[R: TypeInformation](
+ patternFlatSelectFunction: PatternFlatSelectFunction[T, R]): DataStream[R] = {
+ asScalaStream(
+ jPatternStream
+ .flatSelect(patternFlatSelectFunction, implicitly[TypeInformation[R]]))
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contains the resulting events and the resulting
- * timeout events wrapped in a [[Either]] type.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contains the resulting events and the resulting timeout
+ * events wrapped in a [[Either]] type.
+ */
@deprecated
def flatSelect[L: TypeInformation, R: TypeInformation](
- patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
- patternFlatSelectFunction: PatternFlatSelectFunction[T, R])
- : DataStream[Either[L, R]] = {
+ patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
+ patternFlatSelectFunction: PatternFlatSelectFunction[T, R]): DataStream[Either[L, R]] = {
val outputTag = OutputTag[L]("dummy-timeouted")
val mainStream = flatSelect(outputTag, patternFlatTimeoutFunction, patternFlatSelectFunction)
@@ -186,54 +205,57 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream which contains the resulting elements with the resulting timeout elements
- * in a side output.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream which contains the resulting elements with the resulting timeout elements in a
+ * side output.
+ */
def flatSelect[L: TypeInformation, R: TypeInformation](
- outputTag: OutputTag[L],
- patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
- patternFlatSelectFunction: PatternFlatSelectFunction[T, R])
- : DataStream[R] = {
+ outputTag: OutputTag[L],
+ patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
+ patternFlatSelectFunction: PatternFlatSelectFunction[T, R]): DataStream[R] = {
val cleanedSelect = cleanClosure(patternFlatSelectFunction)
val cleanedTimeout = cleanClosure(patternFlatTimeoutFunction)
asScalaStream(
- jPatternStream.flatSelect(
- outputTag,
- cleanedTimeout,
- implicitly[TypeInformation[R]],
- cleanedSelect))
+ jPatternStream
+ .flatSelect(outputTag, cleanedTimeout, implicitly[TypeInformation[R]], cleanedSelect))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
- * one resulting element.
- *
- * @param patternSelectFun The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern select function.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * @param patternSelectFun
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern select function.
+ */
def select[R: TypeInformation](patternSelectFun: Map[String, Iterable[T]] => R): DataStream[R] = {
val cleanFun = cleanClosure(patternSelectFun)
@@ -245,31 +267,34 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting element.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contain the resulting events and resulting timeout
- * events.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting element.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contain the resulting events and resulting timeout events.
+ */
@deprecated
def select[L: TypeInformation, R: TypeInformation](
- patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L) (
- patternSelectFunction: Map[String, Iterable[T]] => R)
- : DataStream[Either[L, R]] = {
+ patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L)(
+ patternSelectFunction: Map[String, Iterable[T]] => R): DataStream[Either[L, R]] = {
val cleanSelectFun = cleanClosure(patternSelectFunction)
val cleanTimeoutFun = cleanClosure(patternTimeoutFunction)
@@ -287,31 +312,34 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting element.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream of either type which contain the resulting events and resulting timeout
- * events.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting element.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream of either type which contain the resulting events and resulting timeout events.
+ */
def select[L: TypeInformation, R: TypeInformation](outputTag: OutputTag[L])(
- patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L) (
- patternSelectFunction: Map[String, Iterable[T]] => R)
- : DataStream[R] = {
+ patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L)(
+ patternSelectFunction: Map[String, Iterable[T]] => R): DataStream[R] = {
val cleanSelectFun = cleanClosure(patternSelectFunction)
val cleanTimeoutFun = cleanClosure(patternTimeoutFunction)
@@ -329,18 +357,19 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function
- * can produce an arbitrary number of resulting elements.
- *
- * @param patternFlatSelectFun The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern flat select
- * function.
- */
- def flatSelect[R: TypeInformation](patternFlatSelectFun: (Map[String, Iterable[T]],
- Collector[R]) => Unit): DataStream[R] = {
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * @param patternFlatSelectFun
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern flat select function.
+ */
+ def flatSelect[R: TypeInformation](
+ patternFlatSelectFun: (Map[String, Iterable[T]], Collector[R]) => Unit): DataStream[R] = {
val cleanFun = cleanClosure(patternFlatSelectFun)
val patternFlatSelectFunction: PatternFlatSelectFunction[T, R] =
@@ -353,31 +382,36 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contains the resulting events and the resulting
- * timeout events wrapped in a [[Either]] type.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contains the resulting events and the resulting timeout
+ * events wrapped in a [[Either]] type.
+ */
@deprecated
def flatSelect[L: TypeInformation, R: TypeInformation](
- patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit) (
+ patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit)(
patternFlatSelectFunction: (Map[String, Iterable[T]], Collector[R]) => Unit)
- : DataStream[Either[L, R]] = {
+ : DataStream[Either[L, R]] = {
val cleanSelectFun = cleanClosure(patternFlatSelectFunction)
val cleanTimeoutFun = cleanClosure(patternFlatTimeoutFunction)
@@ -389,9 +423,9 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
val patternFlatTimeoutFun = new PatternFlatTimeoutFunction[T, L] {
override def timeout(
- pattern: JMap[String, JList[T]],
- timeoutTimestamp: Long, out: Collector[L])
- : Unit = {
+ pattern: JMap[String, JList[T]],
+ timeoutTimestamp: Long,
+ out: Collector[L]): Unit = {
cleanTimeoutFun(mapToScala(pattern), timeoutTimestamp, out)
}
}
@@ -400,31 +434,36 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream of either type which contains the resulting events and the resulting
- * timeout events wrapped in a [[Either]] type.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream of either type which contains the resulting events and the resulting timeout
+ * events wrapped in a [[Either]] type.
+ */
def flatSelect[L: TypeInformation, R: TypeInformation](outputTag: OutputTag[L])(
- patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit) (
- patternFlatSelectFunction: (Map[String, Iterable[T]], Collector[R]) => Unit)
- : DataStream[R] = {
+ patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit)(
+ patternFlatSelectFunction: (Map[String, Iterable[T]], Collector[R]) => Unit)
+ : DataStream[R] = {
val cleanSelectFun = cleanClosure(patternFlatSelectFunction)
val cleanTimeoutFun = cleanClosure(patternFlatTimeoutFunction)
@@ -436,9 +475,9 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
val patternFlatTimeoutFun = new PatternFlatTimeoutFunction[T, L] {
override def timeout(
- pattern: JMap[String, JList[T]],
- timeoutTimestamp: Long, out: Collector[L])
- : Unit = {
+ pattern: JMap[String, JList[T]],
+ timeoutTimestamp: Long,
+ out: Collector[L]): Unit = {
cleanTimeoutFun(mapToScala(pattern), timeoutTimestamp, out)
}
}
@@ -446,7 +485,7 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
flatSelect(outputTag, patternFlatTimeoutFun, patternFlatSelectFun)
}
- def sideOutputLateData(lateDataOutputTag: OutputTag[T]): PatternStream[T] = {
+ def sideOutputLateData(lateDataOutputTag: OutputTag[T]): PatternStream[T] = {
PatternStream[T](jPatternStream.sideOutputLateData(lateDataOutputTag))
}
@@ -460,12 +499,15 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
object PatternStream {
+
/**
- *
- * @param jPatternStream Underlying pattern stream from Java API
- * @tparam T Type of the events
- * @return A new pattern stream wrapping the pattern stream from Java APU
- */
+ * @param jPatternStream
+ * Underlying pattern stream from Java API
+ * @tparam T
+ * Type of the events
+ * @return
+ * A new pattern stream wrapping the pattern stream from Java APU
+ */
def apply[T](jPatternStream: JPatternStream[T]): PatternStream[T] = {
new PatternStream[T](jPatternStream)
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala
index 8bb713397f9..55340c2ad3e 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala
@@ -19,15 +19,15 @@ package org.apache.flink.cep.scala.conditions
import java.io.Serializable
-/**
- * The context used when evaluating the {@link IterativeCondition condition}.
- */
+/** The context used when evaluating the {@link IterativeCondition condition}. */
trait Context[T] extends Serializable {
+
/**
- * @return An { @link Iterable} over the already accepted elements
- * for a given pattern. Elements are iterated in the order they were
- * inserted in the pattern.
- * @param name The name of the pattern.
- */
+ * @return
+ * An { @link Iterable} over the already accepted elements for a given pattern. Elements are
+ * iterated in the order they were inserted in the pattern.
+ * @param name
+ * The name of the pattern.
+ */
def getEventsForPattern(name: String): Iterable[T]
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala
index 223e9c90c6f..52ffae2475f 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala
@@ -17,25 +17,28 @@
*/
package org.apache.flink.cep
-import java.util.{List => JList, Map => JMap}
-
import org.apache.flink.api.scala.ClosureCleaner
import org.apache.flink.cep.{PatternStream => JPatternStream}
+import java.util.{List => JList, Map => JMap}
+
package object scala {
import collection.JavaConverters._
import collection.Map
/**
- * Utility method to wrap [[org.apache.flink.cep.PatternStream]] for usage with the Scala API.
- *
- * @param javaPatternStream The underlying pattern stream from the Java API
- * @tparam T Type of the events
- * @return A pattern stream from the Scala API which wraps a pattern stream from the Java API
- */
- private[flink] def wrapPatternStream[T](javaPatternStream: JPatternStream[T])
- : scala.PatternStream[T] = {
+ * Utility method to wrap [[org.apache.flink.cep.PatternStream]] for usage with the Scala API.
+ *
+ * @param javaPatternStream
+ * The underlying pattern stream from the Java API
+ * @tparam T
+ * Type of the events
+ * @return
+ * A pattern stream from the Scala API which wraps a pattern stream from the Java API
+ */
+ private[flink] def wrapPatternStream[T](
+ javaPatternStream: JPatternStream[T]): scala.PatternStream[T] = {
Option(javaPatternStream) match {
case Some(p) => PatternStream[T](p)
case None =>
@@ -52,4 +55,3 @@ package object scala {
map.asScala.mapValues(_.asScala.toIterable)
}
}
-
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala
index 6210956bfb2..0143625e2d9 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala
@@ -17,24 +17,27 @@
*/
package org.apache.flink.cep.scala.pattern
-import org.apache.flink.cep.pattern.conditions.IterativeCondition
import org.apache.flink.cep.pattern.{GroupPattern => JGroupPattern}
+import org.apache.flink.cep.pattern.conditions.IterativeCondition
/**
- * Base class for a group pattern definition.
- *
- * @param jGroupPattern Underlying Java API GroupPattern
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- */
-class GroupPattern[T , F <: T](jGroupPattern: JGroupPattern[T, F])
- extends Pattern[T , F](jGroupPattern) {
+ * Base class for a group pattern definition.
+ *
+ * @param jGroupPattern
+ * Underlying Java API GroupPattern
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ */
+class GroupPattern[T, F <: T](jGroupPattern: JGroupPattern[T, F])
+ extends Pattern[T, F](jGroupPattern) {
- override def where(condition: IterativeCondition[F] ) =
- throw new UnsupportedOperationException ("GroupPattern does not support where clause.")
+ override def where(condition: IterativeCondition[F]) =
+ throw new UnsupportedOperationException("GroupPattern does not support where clause.")
- override def or(condition: IterativeCondition[F] ) =
- throw new UnsupportedOperationException ("GroupPattern does not support or clause.")
+ override def or(condition: IterativeCondition[F]) =
+ throw new UnsupportedOperationException("GroupPattern does not support or clause.")
override def subtype[S <: F](clazz: Class[S]) =
throw new UnsupportedOperationException("GroupPattern does not support subtype clause.")
@@ -44,13 +47,17 @@ class GroupPattern[T , F <: T](jGroupPattern: JGroupPattern[T, F])
object GroupPattern {
/**
- * Constructs a new GroupPattern by wrapping a given Java API GroupPattern
- *
- * @param jGroupPattern Underlying Java API GroupPattern.
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- * @return New wrapping GroupPattern object
- */
+ * Constructs a new GroupPattern by wrapping a given Java API GroupPattern
+ *
+ * @param jGroupPattern
+ * Underlying Java API GroupPattern.
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ * @return
+ * New wrapping GroupPattern object
+ */
def apply[T, F <: T](jGroupPattern: JGroupPattern[T, F]) = new GroupPattern[T, F](jGroupPattern)
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
index c2d2788c0d8..167603d0823 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
@@ -19,57 +19,49 @@ package org.apache.flink.cep.scala.pattern
import org.apache.flink.cep
import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy
-import org.apache.flink.cep.pattern.conditions.IterativeCondition.{Context => JContext}
+import org.apache.flink.cep.pattern.{MalformedPatternException, Pattern => JPattern, Quantifier}
import org.apache.flink.cep.pattern.conditions.{IterativeCondition, SimpleCondition}
-import org.apache.flink.cep.pattern.{MalformedPatternException, Quantifier, Pattern => JPattern}
+import org.apache.flink.cep.pattern.conditions.IterativeCondition.{Context => JContext}
import org.apache.flink.cep.scala.conditions.Context
import org.apache.flink.streaming.api.windowing.time.Time
/**
- * Base class for a pattern definition.
- *
- * A pattern definition is used by [[org.apache.flink.cep.nfa.compiler.NFACompiler]] to create
- * a [[org.apache.flink.cep.nfa.NFA]].
- *
- * {{{
- * Pattern<T, F> pattern = Pattern.<T>begin("start")
- * .next("middle").subtype(F.class)
- * .followedBy("end").where(new MyCondition());
- * }}}
- *
- * @param jPattern Underlying Java API Pattern
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- */
-class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
+ * Base class for a pattern definition.
+ *
+ * A pattern definition is used by [[org.apache.flink.cep.nfa.compiler.NFACompiler]] to create a
+ * [[org.apache.flink.cep.nfa.NFA]].
+ *
+ * {{{
+ * Pattern<T, F> pattern = Pattern.<T>begin("start")
+ * .next("middle").subtype(F.class)
+ * .followedBy("end").where(new MyCondition());
+ * }}}
+ *
+ * @param jPattern
+ * Underlying Java API Pattern
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ */
+class Pattern[T, F <: T](jPattern: JPattern[T, F]) {
private[flink] def wrappedPattern = jPattern
- /**
- * @return The previous pattern
- */
+ /** @return The previous pattern */
def getPrevious: Option[Pattern[T, _ <: T]] = {
wrapPattern(jPattern.getPrevious)
}
- /**
- *
- * @return Name of the pattern operator
- */
+ /** @return Name of the pattern operator */
def getName: String = jPattern.getName
- /**
- *
- * @return Window length in which the pattern match has to occur
- */
+ /** @return Window length in which the pattern match has to occur */
def getWindowTime: Option[Time] = {
Option(jPattern.getWindowTime)
}
- /**
- *
- * @return currently applied quantifier to this pattern
- */
+ /** @return currently applied quantifier to this pattern */
def getQuantifier: Quantifier = jPattern.getQuantifier
def getCondition: Option[IterativeCondition[F]] = {
@@ -81,30 +73,30 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{AND}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition as an [[IterativeCondition]].
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{AND}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition as an [[IterativeCondition]].
+ * @return
+ * The pattern with the new condition is set.
+ */
def where(condition: IterativeCondition[F]): Pattern[T, F] = {
jPattern.where(condition)
this
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{AND}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition to be set.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{AND}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition to be set.
+ * @return
+ * The pattern with the new condition is set.
+ */
def where(condition: (F, Context[F]) => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -117,15 +109,15 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{AND}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition to be set.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{AND}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition to be set.
+ * @return
+ * The pattern with the new condition is set.
+ */
def where(condition: F => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -136,30 +128,30 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{OR}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition as an [[IterativeCondition]].
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{OR}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition as an [[IterativeCondition]].
+ * @return
+ * The pattern with the new condition is set.
+ */
def or(condition: IterativeCondition[F]): Pattern[T, F] = {
jPattern.or(condition)
this
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{OR}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The {{{OR}}} condition.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{OR}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The {{{OR}}} condition.
+ * @return
+ * The pattern with the new condition is set.
+ */
def or(condition: F => Boolean): Pattern[T, F] = {
val condFun = new SimpleCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -171,15 +163,15 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{OR}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The {{{OR}}} condition.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{OR}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The {{{OR}}} condition.
+ * @return
+ * The pattern with the new condition is set.
+ */
def or(condition: (F, Context[F]) => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -191,37 +183,42 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Applies a subtype constraint on the current pattern. This means that an event has
- * to be of the given subtype in order to be matched.
- *
- * @param clazz Class of the subtype
- * @tparam S Type of the subtype
- * @return The same pattern with the new subtype constraint
- */
+ * Applies a subtype constraint on the current pattern. This means that an event has to be of the
+ * given subtype in order to be matched.
+ *
+ * @param clazz
+ * Class of the subtype
+ * @tparam S
+ * Type of the subtype
+ * @return
+ * The same pattern with the new subtype constraint
+ */
def subtype[S <: F](clazz: Class[S]): Pattern[T, S] = {
jPattern.subtype(clazz)
this.asInstanceOf[Pattern[T, S]]
}
/**
- * Applies a stop condition for a looping state. It allows cleaning the underlying state.
- *
- * @param untilCondition a condition an event has to satisfy to stop collecting events into
- * looping state
- * @return The same pattern with applied untilCondition
- */
+ * Applies a stop condition for a looping state. It allows cleaning the underlying state.
+ *
+ * @param untilCondition
+ * a condition an event has to satisfy to stop collecting events into looping state
+ * @return
+ * The same pattern with applied untilCondition
+ */
def until(untilCondition: IterativeCondition[F]): Pattern[T, F] = {
jPattern.until(untilCondition)
this
}
/**
- * Applies a stop condition for a looping state. It allows cleaning the underlying state.
- *
- * @param untilCondition a condition an event has to satisfy to stop collecting events into
- * looping state
- * @return The same pattern with applied untilCondition
- */
+ * Applies a stop condition for a looping state. It allows cleaning the underlying state.
+ *
+ * @param untilCondition
+ * a condition an event has to satisfy to stop collecting events into looping state
+ * @return
+ * The same pattern with applied untilCondition
+ */
def until(untilCondition: (F, Context[F]) => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(untilCondition)
@@ -233,12 +230,13 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Applies a stop condition for a looping state. It allows cleaning the underlying state.
- *
- * @param untilCondition a condition an event has to satisfy to stop collecting events into
- * looping state
- * @return The same pattern with applied untilCondition
- */
+ * Applies a stop condition for a looping state. It allows cleaning the underlying state.
+ *
+ * @param untilCondition
+ * a condition an event has to satisfy to stop collecting events into looping state
+ * @return
+ * The same pattern with applied untilCondition
+ */
def until(untilCondition: F => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(untilCondition)
@@ -249,240 +247,270 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Defines the maximum time interval in which a matching pattern has to be completed in
- * order to be considered valid. This interval corresponds to the maximum time gap between first
- * and the last event.
- *
- * @param windowTime Time of the matching window
- * @return The same pattern operator with the new window length
- */
+ * Defines the maximum time interval in which a matching pattern has to be completed in order to
+ * be considered valid. This interval corresponds to the maximum time gap between first and the
+ * last event.
+ *
+ * @param windowTime
+ * Time of the matching window
+ * @return
+ * The same pattern operator with the new window length
+ */
def within(windowTime: Time): Pattern[T, F] = {
jPattern.within(windowTime)
this
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces strict
- * temporal contiguity. This means that the whole pattern sequence matches only
- * if an event which matches this pattern directly follows the preceding matching
- * event. Thus, there cannot be any events in between two matching events.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces strict temporal contiguity.
+ * This means that the whole pattern sequence matches only if an event which matches this pattern
+ * directly follows the preceding matching event. Thus, there cannot be any events in between two
+ * matching events.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def next(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.next(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces that there is no event
- * matching this pattern right after the preceding matched event.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces that there is no event
+ * matching this pattern right after the preceding matched event.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def notNext(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.notNext(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedBy(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.followedBy(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces that there is no event
- * matching this pattern between the preceding pattern and succeeding this one.
- *
- * NOTE: There has to be other pattern after this one.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
- def notFollowedBy(name : String): Pattern[T, T] = {
+ * Appends a new pattern to the existing one. The new pattern enforces that there is no event
+ * matching this pattern between the preceding pattern and succeeding this one.
+ *
+ * NOTE: There has to be other pattern after this one.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
+ def notFollowedBy(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.notFollowedBy(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedByAny(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.followedByAny(name))
}
-
/**
- * Specifies that this pattern is optional for a final match of the pattern
- * sequence to happen.
- *
- * @return The same pattern as optional.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern is optional for a final match of the pattern sequence to happen.
+ *
+ * @return
+ * The same pattern as optional.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def optional: Pattern[T, F] = {
jPattern.optional()
this
}
/**
- * Specifies that this pattern can occur {{{one or more}}} times.
- * This means at least one and at most infinite number of events can
- * be matched to this pattern.
- *
- * If this quantifier is enabled for a
- * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
- * {{{A1 A2 B}}} appears, this will generate patterns:
- * {{{A1 B}}} and {{{A1 A2 B}}}. See also {{{allowCombinations()}}}.
- *
- * @return The same pattern with a [[Quantifier.looping()]] quantifier applied.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern can occur {{{one or more}}} times. This means at least one and at
+ * most infinite number of events can be matched to this pattern.
+ *
+ * If this quantifier is enabled for a pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of
+ * events {{{A1 A2 B}}} appears, this will generate patterns: {{{A1 B}}} and {{{A1 A2 B}}}. See
+ * also {{{allowCombinations()}}}.
+ *
+ * @return
+ * The same pattern with a [[Quantifier.looping()]] quantifier applied.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def oneOrMore: Pattern[T, F] = {
jPattern.oneOrMore()
this
}
/**
- * Specifies that this pattern is greedy.
- * This means as many events as possible will be matched to this pattern.
- *
- * @return The same pattern with { @link Quantifier#greedy} set to true.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern is greedy. This means as many events as possible will be matched to
+ * this pattern.
+ *
+ * @return
+ * The same pattern with { @link Quantifier#greedy} set to true.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def greedy: Pattern[T, F] = {
jPattern.greedy()
this
}
/**
- * Specifies exact number of times that this pattern should be matched.
- *
- * @param times number of times matching event must appear
- * @return The same pattern with number of times applied
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies exact number of times that this pattern should be matched.
+ *
+ * @param times
+ * number of times matching event must appear
+ * @return
+ * The same pattern with number of times applied
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def times(times: Int): Pattern[T, F] = {
jPattern.times(times)
this
}
/**
- * Specifies that the pattern can occur between from and to times.
- *
- * @param from number of times matching event must appear at least
- * @param to number of times matching event must appear at most
- * @return The same pattern with the number of times range applied
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that the pattern can occur between from and to times.
+ *
+ * @param from
+ * number of times matching event must appear at least
+ * @param to
+ * number of times matching event must appear at most
+ * @return
+ * The same pattern with the number of times range applied
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def times(from: Int, to: Int): Pattern[T, F] = {
jPattern.times(from, to)
this
}
/**
- * Specifies that this pattern can occur the specified times at least.
- * This means at least the specified times and at most infinite number of events can
- * be matched to this pattern.
- *
- * @return The same pattern with a { @link Quantifier#looping(ConsumingStrategy)} quantifier
- * applied.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern can occur the specified times at least. This means at least the
+ * specified times and at most infinite number of events can be matched to this pattern.
+ *
+ * @return
+ * The same pattern with a { @link Quantifier#looping(ConsumingStrategy)} quantifier applied.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def timesOrMore(times: Int): Pattern[T, F] = {
jPattern.timesOrMore(times)
this
}
/**
- * Applicable only to [[Quantifier.looping()]] and [[Quantifier.times()]] patterns,
- * this option allows more flexibility to the matching events.
- *
- * If {{{allowCombinations()}}} is not applied for a
- * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
- * {{{A1 A2 B}}} appears, this will generate patterns:
- * {{{A1 B}}} and {{{A1 A2 B}}}. If this method is applied, we
- * will have {{{A1 B}}}, {{{A2 B}}} and {{{A1 A2 B}}}.
- *
- * @return The same pattern with the updated quantifier.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Applicable only to [[Quantifier.looping()]] and [[Quantifier.times()]] patterns, this option
+ * allows more flexibility to the matching events.
+ *
+ * If {{{allowCombinations()}}} is not applied for a pattern {{{A.oneOrMore().followedBy(B)}}} and
+ * a sequence of events {{{A1 A2 B}}} appears, this will generate patterns: {{{A1 B}}} and
+ * {{{A1 A2 B}}}. If this method is applied, we will have {{{A1 B}}}, {{{A2 B}}} and
+ * {{{A1 A2 B}}}.
+ *
+ * @return
+ * The same pattern with the updated quantifier.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def allowCombinations(): Pattern[T, F] = {
jPattern.allowCombinations()
this
}
/**
- * Works in conjunction with [[Pattern#oneOrMore()]] or [[Pattern#times(int)]].
- * Specifies that any not matching element breaks the loop.
- *
- * E.g. a pattern like:
- * {{{
- * Pattern.begin("start").where(_.getName().equals("c"))
- * .followedBy("middle").where(_.getName().equals("a")).oneOrMore().consecutive()
- * .followedBy("end1").where(_.getName().equals("b"));
- * }}}
- *
- * For a sequence: C D A1 A2 A3 D A4 B
- *
- * will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
- *
- * By default a relaxed continuity is applied.
- * @return pattern with continuity changed to strict
- */
+ * Works in conjunction with [[Pattern#oneOrMore()]] or [[Pattern#times(int)]]. Specifies that any
+ * not matching element breaks the loop.
+ *
+ * E.g. a pattern like:
+ * {{{
+ * Pattern.begin("start").where(_.getName().equals("c"))
+ * .followedBy("middle").where(_.getName().equals("a")).oneOrMore().consecutive()
+ * .followedBy("end1").where(_.getName().equals("b"));
+ * }}}
+ *
+ * For a sequence: C D A1 A2 A3 D A4 B
+ *
+ * will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
+ *
+ * By default a relaxed continuity is applied.
+ * @return
+ * pattern with continuity changed to strict
+ */
def consecutive(): Pattern[T, F] = {
jPattern.consecutive()
this
}
/**
- * Appends a new group pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param pattern the pattern to append
- * @return A new pattern which is appended to this one
- */
+ * Appends a new group pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param pattern
+ * the pattern to append
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedBy(pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](jPattern.followedBy(pattern.wrappedPattern))
/**
- * Appends a new group pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param pattern the pattern to append
- * @return A new pattern which is appended to this one
- */
+ * Appends a new group pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param pattern
+ * the pattern to append
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedByAny(pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](jPattern.followedByAny(pattern.wrappedPattern))
/**
- * Appends a new group pattern to the existing one. The new pattern enforces strict
- * temporal contiguity. This means that the whole pattern sequence matches only
- * if an event which matches this pattern directly follows the preceding matching
- * event. Thus, there cannot be any events in between two matching events.
- *
- * @param pattern the pattern to append
- * @return A new pattern which is appended to this one
- */
+ * Appends a new group pattern to the existing one. The new pattern enforces strict temporal
+ * contiguity. This means that the whole pattern sequence matches only if an event which matches
+ * this pattern directly follows the preceding matching event. Thus, there cannot be any events in
+ * between two matching events.
+ *
+ * @param pattern
+ * the pattern to append
+ * @return
+ * A new pattern which is appended to this one
+ */
def next(pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](jPattern.next(pattern.wrappedPattern))
/**
- * Get after match skip strategy.
- * @return current after match skip strategy
- */
+ * Get after match skip strategy.
+ * @return
+ * current after match skip strategy
+ */
def getAfterMatchSkipStrategy: AfterMatchSkipStrategy =
jPattern.getAfterMatchSkipStrategy
}
@@ -490,56 +518,71 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
object Pattern {
/**
- * Constructs a new Pattern by wrapping a given Java API Pattern
- *
- * @param jPattern Underlying Java API Pattern.
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- * @return New wrapping Pattern object
- */
+ * Constructs a new Pattern by wrapping a given Java API Pattern
+ *
+ * @param jPattern
+ * Underlying Java API Pattern.
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ * @return
+ * New wrapping Pattern object
+ */
def apply[T, F <: T](jPattern: JPattern[T, F]) = new Pattern[T, F](jPattern)
/**
- * Starts a new pattern sequence. The provided name is the one of the initial pattern
- * of the new sequence. Furthermore, the base type of the event sequence is set.
- *
- * @param name The name of starting pattern of the new pattern sequence
- * @tparam X Base type of the event pattern
- * @return The first pattern of a pattern sequence
- */
+ * Starts a new pattern sequence. The provided name is the one of the initial pattern of the new
+ * sequence. Furthermore, the base type of the event sequence is set.
+ *
+ * @param name
+ * The name of starting pattern of the new pattern sequence
+ * @tparam X
+ * Base type of the event pattern
+ * @return
+ * The first pattern of a pattern sequence
+ */
def begin[X](name: String): Pattern[X, X] = Pattern(JPattern.begin(name))
/**
- * Starts a new pattern sequence. The provided name is the one of the initial pattern
- * of the new sequence. Furthermore, the base type of the event sequence is set.
- *
- * @param name The name of starting pattern of the new pattern sequence
- * @param afterMatchSkipStrategy The skip strategy to use after each match
- * @tparam X Base type of the event pattern
- * @return The first pattern of a pattern sequence
- */
+ * Starts a new pattern sequence. The provided name is the one of the initial pattern of the new
+ * sequence. Furthermore, the base type of the event sequence is set.
+ *
+ * @param name
+ * The name of starting pattern of the new pattern sequence
+ * @param afterMatchSkipStrategy
+ * The skip strategy to use after each match
+ * @tparam X
+ * Base type of the event pattern
+ * @return
+ * The first pattern of a pattern sequence
+ */
def begin[X](name: String, afterMatchSkipStrategy: AfterMatchSkipStrategy): Pattern[X, X] =
Pattern(JPattern.begin(name, afterMatchSkipStrategy))
/**
- * Starts a new pattern sequence. The provided pattern is the initial pattern
- * of the new sequence.
- *
- * @param pattern the pattern to begin with
- * @return the first pattern of a pattern sequence
- */
+ * Starts a new pattern sequence. The provided pattern is the initial pattern of the new sequence.
+ *
+ * @param pattern
+ * the pattern to begin with
+ * @return
+ * the first pattern of a pattern sequence
+ */
def begin[T, F <: T](pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](JPattern.begin(pattern.wrappedPattern))
/**
- * Starts a new pattern sequence. The provided pattern is the initial pattern
- * of the new sequence.
- *
- * @param pattern the pattern to begin with
- * @param afterMatchSkipStrategy The skip strategy to use after each match
- * @return The first pattern of a pattern sequence
- */
- def begin[T, F <: T](pattern: Pattern[T, F],
+ * Starts a new pattern sequence. The provided pattern is the initial pattern of the new sequence.
+ *
+ * @param pattern
+ * the pattern to begin with
+ * @param afterMatchSkipStrategy
+ * The skip strategy to use after each match
+ * @return
+ * The first pattern of a pattern sequence
+ */
+ def begin[T, F <: T](
+ pattern: Pattern[T, F],
afterMatchSkipStrategy: AfterMatchSkipStrategy): GroupPattern[T, F] =
GroupPattern(JPattern.begin(pattern.wrappedPattern, afterMatchSkipStrategy))
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
index 13305dd0413..0c3dab9b88a 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
@@ -20,29 +20,35 @@ package org.apache.flink.cep.scala
import org.apache.flink.cep.pattern.{Pattern => JPattern}
import org.apache.flink.cep.pattern.conditions.IterativeCondition.{Context => JContext}
import org.apache.flink.cep.scala.conditions.Context
+
import scala.collection.JavaConverters._
package object pattern {
+
/**
- * Utility method to wrap [[org.apache.flink.cep.pattern.Pattern]] and its subclasses
- * for usage with the Scala API.
- *
- * @param javaPattern The underlying pattern from the Java API
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- * @return A pattern from the Scala API which wraps the pattern from the Java API
- */
- private[flink] def wrapPattern[T, F <: T](javaPattern: JPattern[T, F])
- : Option[Pattern[T, F]] = javaPattern match {
- case p: JPattern[T, F] => Some(Pattern[T, F](p))
- case _ => None
- }
+ * Utility method to wrap [[org.apache.flink.cep.pattern.Pattern]] and its subclasses for usage
+ * with the Scala API.
+ *
+ * @param javaPattern
+ * The underlying pattern from the Java API
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ * @return
+ * A pattern from the Scala API which wraps the pattern from the Java API
+ */
+ private[flink] def wrapPattern[T, F <: T](javaPattern: JPattern[T, F]): Option[Pattern[T, F]] =
+ javaPattern match {
+ case p: JPattern[T, F] => Some(Pattern[T, F](p))
+ case _ => None
+ }
private[pattern] class JContextWrapper[F](private val jContext: JContext[F])
- extends Context[F] with Serializable {
+ extends Context[F]
+ with Serializable {
override def getEventsForPattern(name: String): Iterable[F] =
jContext.getEventsForPattern(name).asScala
}
}
-
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala
index 560d99af4a6..01fab9908d3 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala
@@ -17,17 +17,18 @@
*/
package org.apache.flink.cep.scala
-import java.lang.reflect.Method
-
import org.apache.flink.api.scala.completeness.ScalaAPICompletenessTestBase
import org.apache.flink.cep.{PatternStream => JPatternStream}
+
import org.junit.Test
+import java.lang.reflect.Method
+
import scala.language.existentials
/**
- * This checks whether the CEP Scala API is up to feature parity with the Java API.
- * Implements the [[ScalaAPICompletenessTestBase]] for CEP.
+ * This checks whether the CEP Scala API is up to feature parity with the Java API. Implements the
+ * [[ScalaAPICompletenessTestBase]] for CEP.
*/
class CEPScalaAPICompletenessTest extends ScalaAPICompletenessTestBase {
@@ -39,7 +40,9 @@ class CEPScalaAPICompletenessTest extends ScalaAPICompletenessTestBase {
@Test
override def testCompleteness(): Unit = {
- checkMethods("PatternStream", "PatternStream",
+ checkMethods(
+ "PatternStream",
+ "PatternStream",
classOf[JPatternStream[_]],
classOf[PatternStream[_]])
}
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala
index 80d96aca175..a8f1921bce7 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala
@@ -17,8 +17,6 @@
*/
package org.apache.flink.cep.scala
-import java.lang
-
import org.apache.flink.api.common.functions.RuntimeContext
import org.apache.flink.api.common.functions.util.{FunctionUtils, ListCollector}
import org.apache.flink.cep.functions.{PatternProcessFunction, TimedOutPartialMatchHandler}
@@ -29,13 +27,16 @@ import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.transformations.OneInputTransformation
import org.apache.flink.util
import org.apache.flink.util.{Collector, TestLogger}
+
import org.junit.Assert._
import org.junit.Test
import org.mockito.Mockito
+import java.lang
+
+import scala.collection.{mutable, Map}
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
-import scala.collection.{Map, mutable}
class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
@@ -48,18 +49,19 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
val pStream: PatternStream[(Int, Int)] = CEP.pattern(dummyDataStream, pattern)
val param = Map("begin" -> List((1, 2)))
val result: DataStream[(Int, Int)] = pStream
- .select((pattern: Map[String, Iterable[(Int, Int)]]) => {
- //verifies input parameter forwarding
- assertEquals(param, pattern)
- param("begin").head
- })
+ .select(
+ (pattern: Map[String, Iterable[(Int, Int)]]) => {
+ // verifies input parameter forwarding
+ assertEquals(param, pattern)
+ param("begin").head
+ })
val outList = new java.util.ArrayList[(Int, Int)]
val outParam = new ListCollector[(Int, Int)](outList)
val fun = extractFun[(Int, Int), (Int, Int)](result)
fun.processMatch(param.mapValues(_.asJava).asJava, new ListTestContext, outParam)
- //verifies output parameter forwarding
+ // verifies output parameter forwarding
assertEquals(param("begin").head, outList.get(0))
}
@@ -76,16 +78,16 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
val outParam = new ListCollector[List[Int]](outList)
val result: DataStream[List[Int]] = pStream
-
- .flatSelect((pattern: Map[String, Iterable[List[Int]]], out: Collector[List[Int]]) => {
- //verifies input parameter forwarding
- assertEquals(inParam, pattern)
- out.collect(pattern("begin").head)
- })
+ .flatSelect(
+ (pattern: Map[String, Iterable[List[Int]]], out: Collector[List[Int]]) => {
+ // verifies input parameter forwarding
+ assertEquals(inParam, pattern)
+ out.collect(pattern("begin").head)
+ })
val fun = extractFun[List[Int], List[Int]](result)
fun.processMatch(inParam.mapValues(_.asJava).asJava, new ListTestContext, outParam)
- //verify output parameter forwarding and that flatMap function was actually called
+ // verify output parameter forwarding and that flatMap function was actually called
assertEquals(inList, outList.get(0))
}
@@ -102,23 +104,26 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
val outputTag = OutputTag[Either[String, String]]("timeouted")
val result: DataStream[Either[String, String]] = pStream.flatSelect(outputTag) {
- (pattern: Map[String, Iterable[String]], timestamp: Long,
- out: Collector[Either[String, String]]) =>
- out.collect(Left("timeout"))
- out.collect(Left(pattern("begin").head))
- } {
- (pattern: Map[String, Iterable[String]], out: Collector[Either[String, String]]) =>
- //verifies input parameter forwarding
- assertEquals(inParam, pattern)
- out.collect(Right("match"))
- out.collect(Right(pattern("begin").head))
- }
+ (
+ pattern: Map[String, Iterable[String]],
+ timestamp: Long,
+ out: Collector[Either[String, String]]) =>
+ out.collect(Left("timeout"))
+ out.collect(Left(pattern("begin").head))
+ } {
+ (pattern: Map[String, Iterable[String]], out: Collector[Either[String, String]]) =>
+ // verifies input parameter forwarding
+ assertEquals(inParam, pattern)
+ out.collect(Right("match"))
+ out.collect(Right(pattern("begin").head))
+ }
val fun = extractFun[String, Either[String, String]](result)
val ctx = new ListTestContext
fun.processMatch(inParam.mapValues(_.asJava).asJava, ctx, output)
- fun.asInstanceOf[TimedOutPartialMatchHandler[String]]
+ fun
+ .asInstanceOf[TimedOutPartialMatchHandler[String]]
.processTimedOutMatch(inParam.mapValues(_.asJava).asJava, ctx)
assertEquals(List(Right("match"), Right("barfoo")).asJava, outList)
@@ -126,8 +131,7 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
}
def extractFun[IN, OUT](dataStream: DataStream[OUT]): PatternProcessFunction[IN, OUT] = {
- val oper = dataStream.javaStream
- .getTransformation
+ val oper = dataStream.javaStream.getTransformation
.asInstanceOf[OneInputTransformation[_, _]]
.getOperator
.asInstanceOf[CepOperator[IN, Byte, OUT]]
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala
index ac72bdffb3b..941553eadc7 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala
@@ -17,17 +17,18 @@
*/
package org.apache.flink.cep.scala.pattern
-import java.lang.reflect.Method
-
import org.apache.flink.api.scala.completeness.ScalaAPICompletenessTestBase
import org.apache.flink.cep.pattern.{Pattern => JPattern}
+
import org.junit.Test
+import java.lang.reflect.Method
+
import scala.language.existentials
/**
- * This checks whether the CEP Scala API is up to feature parity with the Java API.
- * Implements the [[ScalaAPICompletenessTestBase]] for CEP.
+ * This checks whether the CEP Scala API is up to feature parity with the Java API. Implements the
+ * [[ScalaAPICompletenessTestBase]] for CEP.
*/
class PatternScalaAPICompletenessTest extends ScalaAPICompletenessTestBase {
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
index 91a15032a91..23453c5d167 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
@@ -17,28 +17,28 @@
*/
package org.apache.flink.cep.scala.pattern
-import org.apache.flink.cep.pattern.{Pattern => JPattern}
-import org.junit.Assert._
-import org.junit.Test
import org.apache.flink.cep.Event
import org.apache.flink.cep.SubEvent
+import org.apache.flink.cep.pattern.{Pattern => JPattern}
import org.apache.flink.cep.pattern.Quantifier.ConsumingStrategy
import org.apache.flink.cep.pattern.conditions._
import org.apache.flink.cep.scala.conditions.Context
+import org.junit.Assert._
+import org.junit.Test
+
class PatternTest {
/**
- * These tests simply check that the pattern construction completes without failure and that the
- * Scala API pattern is synchronous with its wrapped Java API pattern.
- */
+ * These tests simply check that the pattern construction completes without failure and that the
+ * Scala API pattern is synchronous with its wrapped Java API pattern.
+ */
@Test
def testStrictContiguity(): Unit = {
val pattern = Pattern.begin[Event]("start").next("next").next("end")
val jPattern = JPattern.begin[Event]("start").next("next").next("end")
-
assertTrue(checkCongruentRepresentations(pattern, jPattern))
assertTrue(checkCongruentRepresentations(wrapPattern(jPattern).get, jPattern))
@@ -55,7 +55,6 @@ class PatternTest {
assertEquals(preprevious.getName, "start")
}
-
@Test
def testNonStrictContiguity(): Unit = {
val pattern = Pattern.begin[Event]("start").followedBy("next").followedBy("end")
@@ -80,20 +79,23 @@ class PatternTest {
@Test
def testStrictContiguityWithCondition(): Unit = {
- val pattern = Pattern.begin[Event]("start")
+ val pattern = Pattern
+ .begin[Event]("start")
.next("next")
.where((value: Event, _: Context[Event]) => value.getName == "foobar")
.next("end")
.where((value: Event, _: Context[Event]) => value.getId == 42)
- val jPattern = JPattern.begin[Event]("start")
+ val jPattern = JPattern
+ .begin[Event]("start")
.next("next")
.where(new SimpleCondition[Event]() {
@throws[Exception]
def filter(value: Event): Boolean = {
value.getName == "foobar"
}
- }).next("end")
+ })
+ .next("end")
.where(new SimpleCondition[Event]() {
@throws[Exception]
def filter(value: Event): Boolean = {
@@ -122,12 +124,14 @@ class PatternTest {
@Test
def testPatternWithSubtyping(): Unit = {
- val pattern = Pattern.begin[Event]("start")
+ val pattern = Pattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.followedBy("end")
- val jPattern = JPattern.begin[Event]("start")
+ val jPattern = JPattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.followedBy("end")
@@ -152,13 +156,15 @@ class PatternTest {
@Test
def testPatternWithSubtypingAndFilter(): Unit = {
- val pattern = Pattern.begin[Event]("start")
+ val pattern = Pattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.where(_ => false)
.followedBy("end")
- val jpattern = JPattern.begin[Event]("start")
+ val jpattern = JPattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.where(new SimpleCondition[SubEvent]() {
@@ -166,12 +172,12 @@ class PatternTest {
def filter(value: SubEvent): Boolean = {
false
}
- }).followedBy("end")
+ })
+ .followedBy("end")
assertTrue(checkCongruentRepresentations(pattern, jpattern))
assertTrue(checkCongruentRepresentations(wrapPattern(jpattern).get, jpattern))
-
val previous = pattern.getPrevious.orNull
val preprevious = previous.getPrevious.orNull
@@ -187,61 +193,58 @@ class PatternTest {
assertEquals(preprevious.getName, "start")
}
- def checkCongruentRepresentations[T, _ <: T](pattern: Pattern[T, _ <: T],
- jPattern: JPattern[T, _ <: T]): Boolean = {
+ def checkCongruentRepresentations[T, _ <: T](
+ pattern: Pattern[T, _ <: T],
+ jPattern: JPattern[T, _ <: T]): Boolean = {
((pattern == null && jPattern == null)
- || (pattern != null && jPattern != null)
- //check equal pattern names
- && threeWayEquals(
- pattern.getName,
- pattern.wrappedPattern.getName,
- jPattern.getName)
- //check equal time windows
- && threeWayEquals(
+ || (pattern != null && jPattern != null)
+ // check equal pattern names
+ && threeWayEquals(pattern.getName, pattern.wrappedPattern.getName, jPattern.getName)
+ // check equal time windows
+ && threeWayEquals(
pattern.getWindowTime.orNull,
pattern.wrappedPattern.getWindowTime,
jPattern.getWindowTime)
- //check congruent class names / types
- && threeWayEquals(
+ // check congruent class names / types
+ && threeWayEquals(
pattern.getClass.getSimpleName,
pattern.wrappedPattern.getClass.getSimpleName,
jPattern.getClass.getSimpleName)
- //best effort to confirm congruent filter functions
- && compareFilterFunctions(
- pattern.getCondition.orNull,
- jPattern.getCondition)
- //recursively check previous patterns
- && checkCongruentRepresentations(
- pattern.getPrevious.orNull,
- jPattern.getPrevious))
+ // best effort to confirm congruent filter functions
+ && compareFilterFunctions(pattern.getCondition.orNull, jPattern.getCondition)
+ // recursively check previous patterns
+ && checkCongruentRepresentations(pattern.getPrevious.orNull, jPattern.getPrevious))
}
def threeWayEquals(a: AnyRef, b: AnyRef, c: AnyRef): Boolean = {
a == b && b == c
}
- def compareFilterFunctions(sFilter: IterativeCondition[_],
- jFilter: IterativeCondition[_]): Boolean = {
+ def compareFilterFunctions(
+ sFilter: IterativeCondition[_],
+ jFilter: IterativeCondition[_]): Boolean = {
+
/**
- * We would like to simply compare the filter functions like this:
- *
- * {{{(pattern.getFilterFunction.orNull == jPattern.getFilterFunction)}}}
- *
- * However, the closure cleaning makes comparing filter functions by reference impossible.
- * Testing for functional equivalence is an undecidable problem. Thus, for do a best effort by
- * simply matching the presence/absence and known classes of filter functions in the patterns.
- */
+ * We would like to simply compare the filter functions like this:
+ *
+ * {{{(pattern.getFilterFunction.orNull == jPattern.getFilterFunction)}}}
+ *
+ * However, the closure cleaning makes comparing filter functions by reference impossible.
+ * Testing for functional equivalence is an undecidable problem. Thus, for do a best effort by
+ * simply matching the presence/absence and known classes of filter functions in the patterns.
+ */
(sFilter, jFilter) match {
- //matching types: and-filter; branch and recurse for inner filters
- case (saf: AndCondition[_], jaf: AndCondition[_])
- => (compareFilterFunctions(saf.getLeft, jaf.getLeft)
- && compareFilterFunctions(saf.getRight, jaf.getRight))
- //matching types: subtype-filter
+ // matching types: and-filter; branch and recurse for inner filters
+ case (saf: AndCondition[_], jaf: AndCondition[_]) => (
+ compareFilterFunctions(saf.getLeft, jaf.getLeft)
+ && compareFilterFunctions(saf.getRight, jaf.getRight)
+ )
+ // matching types: subtype-filter
case (_: SubtypeCondition[_], _: SubtypeCondition[_]) => true
- //mismatch: one-sided and/subtype-filter
+ // mismatch: one-sided and/subtype-filter
case (_: AndCondition[_] | _: SubtypeCondition[_], _) => false
case (_, _: AndCondition[_] | _: SubtypeCondition[_]) => false
- //from here we can only check mutual presence or absence of a function
+ // from here we can only check mutual presence or absence of a function
case (_: IterativeCondition[_], _: IterativeCondition[_]) => true
case (null, null) => true
case _ => false
diff --git a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala
index d3f2f92199e..d5101c22559 100644
--- a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala
+++ b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala
@@ -15,11 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.examples
-import java.lang.Long
-
import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.scala._
import org.apache.flink.graph.Edge
@@ -28,25 +25,21 @@ import org.apache.flink.graph.library.GSAConnectedComponents
import org.apache.flink.graph.scala._
import org.apache.flink.types.NullValue
+import java.lang.Long
+
/**
- * This example shows how to use Gelly's library methods.
- * You can find all available library methods in [[org.apache.flink.graph.library]].
- *
- * In particular, this example uses the
- * [[GSAConnectedComponents]]
- * library method to compute the connected components of the input graph.
+ * This example shows how to use Gelly's library methods. You can find all available library methods
+ * in [[org.apache.flink.graph.library]].
+ *
+ * In particular, this example uses the [[GSAConnectedComponents]] library method to compute the
+ * connected components of the input graph.
*
- * The input file is a plain text file and must be formatted as follows:
- * Edges are represented by tuples of srcVertexId, trgVertexId which are
- * separated by tabs. Edges themselves are separated by newlines.
- * For example: <code>1\t2\n1\t3\n</code> defines two edges,
- * 1-2 and 1-3.
+ * The input file is a plain text file and must be formatted as follows: Edges are represented by
+ * tuples of srcVertexId, trgVertexId which are separated by tabs. Edges themselves are separated by
+ * newlines. For example: <code>1\t2\n1\t3\n</code> defines two edges, 1-2 and 1-3.
*
- * Usage {{
- * ConnectedComponents <edge path> <result path> <number of iterations>
- * }}
- * If no parameters are provided, the program is run with default data from
- * [[ConnectedComponentsDefaultData]]
+ * Usage {{ ConnectedComponents <edge path> <result path> <number of iterations> }} If no parameters
+ * are provided, the program is run with default data from [[ConnectedComponentsDefaultData]]
*/
object ConnectedComponents {
def main(args: Array[String]) {
@@ -60,7 +53,6 @@ object ConnectedComponents {
val components = graph.run(new GSAConnectedComponents[Long, Long, NullValue](maxIterations))
-
// emit result
if (fileOutput) {
components.writeAsCsv(outputPath, "\n", ",")
@@ -70,7 +62,7 @@ object ConnectedComponents {
}
}
- private final class InitVertices extends MapFunction[Long, Long] {
+ final private class InitVertices extends MapFunction[Long, Long] {
override def map(id: Long) = id
}
@@ -78,44 +70,47 @@ object ConnectedComponents {
// UTIL METHODS
// ***********************************************************************
- private var fileOutput = false
- private var edgesInputPath: String = null
- private var outputPath: String = null
- private var maxIterations = ConnectedComponentsDefaultData.MAX_ITERATIONS
+ private var fileOutput = false
+ private var edgesInputPath: String = null
+ private var outputPath: String = null
+ private var maxIterations = ConnectedComponentsDefaultData.MAX_ITERATIONS
- private def parseParameters(args: Array[String]): Boolean = {
- if(args.length > 0) {
- if(args.length != 3) {
- System.err.println("Usage ConnectedComponents <edge path> <output path> " +
+ private def parseParameters(args: Array[String]): Boolean = {
+ if (args.length > 0) {
+ if (args.length != 3) {
+ System.err.println(
+ "Usage ConnectedComponents <edge path> <output path> " +
"<num iterations>")
- }
- fileOutput = true
- edgesInputPath = args(0)
- outputPath = args(1)
- maxIterations = 2
- } else {
- System.out.println("Executing ConnectedComponents example with default parameters" +
+ }
+ fileOutput = true
+ edgesInputPath = args(0)
+ outputPath = args(1)
+ maxIterations = 2
+ } else {
+ System.out.println(
+ "Executing ConnectedComponents example with default parameters" +
" and built-in default data.")
- System.out.println(" Provide parameters to read input data from files.")
- System.out.println(" See the documentation for the correct format of input files.")
- System.out.println("Usage ConnectedComponents <edge path> <output path> " +
+ System.out.println(" Provide parameters to read input data from files.")
+ System.out.println(" See the documentation for the correct format of input files.")
+ System.out.println(
+ "Usage ConnectedComponents <edge path> <output path> " +
"<num iterations>")
- }
- true
}
+ true
+ }
- private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, NullValue]] = {
- if (fileOutput) {
- env.readCsvFile[(Long, Long)](edgesInputPath,
- lineDelimiter = "\n",
- fieldDelimiter = "\t")
- .map(edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
- } else {
- val edgeData = ConnectedComponentsDefaultData.DEFAULT_EDGES map {
- case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
- }
- env.fromCollection(edgeData).map(
- edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
+ private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, NullValue]] = {
+ if (fileOutput) {
+ env
+ .readCsvFile[(Long, Long)](edgesInputPath, lineDelimiter = "\n", fieldDelimiter = "\t")
+ .map(edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
+ } else {
+ val edgeData = ConnectedComponentsDefaultData.DEFAULT_EDGES.map {
+ case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
}
+ env
+ .fromCollection(edgeData)
+ .map(edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
}
+ }
}
diff --git a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala
index bd5c63bb4f2..672868648b5 100644
--- a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala
+++ b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.examples
import org.apache.flink.api.common.functions.MapFunction
@@ -28,14 +27,13 @@ import org.apache.flink.graph.scala.utils.Tuple3ToEdgeMap
/**
* This example shows how to use Gelly's gather-sum-apply iterations.
- *
- * It is an implementation of the Single-Source-Shortest-Paths algorithm.
*
- * The input file is a plain text file and must be formatted as follows:
- * Edges are represented by tuples of srcVertexId, trgVertexId, distance which are
- * separated by tabs. Edges themselves are separated by newlines.
- * For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges,
- * edge 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
+ * It is an implementation of the Single-Source-Shortest-Paths algorithm.
+ *
+ * The input file is a plain text file and must be formatted as follows: Edges are represented by
+ * tuples of srcVertexId, trgVertexId, distance which are separated by tabs. Edges themselves are
+ * separated by newlines. For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges, edge
+ * 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
*
* If no parameters are provided, the program is run with default data from
* [[SingleSourceShortestPathsData]]
@@ -51,8 +49,11 @@ object GSASingleSourceShortestPaths {
val graph = Graph.fromDataSet[Long, Double, Double](edges, new InitVertices(srcVertexId), env)
// Execute the gather-sum-apply iteration
- val result = graph.runGatherSumApplyIteration(new CalculateDistances, new ChooseMinDistance,
- new UpdateDistance, maxIterations)
+ val result = graph.runGatherSumApplyIteration(
+ new CalculateDistances,
+ new ChooseMinDistance,
+ new UpdateDistance,
+ maxIterations)
// Extract the vertices as the result
val singleSourceShortestPaths = result.getVertices
@@ -70,7 +71,7 @@ object GSASingleSourceShortestPaths {
// Single Source Shortest Path UDFs
// --------------------------------------------------------------------------------------------
- private final class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
+ final private class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
override def map(id: Long) = {
if (id.equals(srcId)) {
@@ -81,19 +82,19 @@ object GSASingleSourceShortestPaths {
}
}
- private final class CalculateDistances extends GatherFunction[Double, Double, Double] {
+ final private class CalculateDistances extends GatherFunction[Double, Double, Double] {
override def gather(neighbor: Neighbor[Double, Double]) = {
neighbor.getNeighborValue + neighbor.getEdgeValue
}
}
- private final class ChooseMinDistance extends SumFunction[Double, Double, Double] {
+ final private class ChooseMinDistance extends SumFunction[Double, Double, Double] {
override def sum(newValue: Double, currentValue: Double) = {
Math.min(newValue, currentValue)
}
}
- private final class UpdateDistance extends ApplyFunction[Long, Double, Double] {
+ final private class UpdateDistance extends ApplyFunction[Long, Double, Double] {
override def apply(newDistance: Double, oldDistance: Double) = {
if (newDistance < oldDistance) {
setResult(newDistance)
@@ -112,10 +113,11 @@ object GSASingleSourceShortestPaths {
private var maxIterations = 5
private def parseParameters(args: Array[String]): Boolean = {
- if(args.length > 0) {
- if(args.length != 4) {
- System.err.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ if (args.length > 0) {
+ if (args.length != 4) {
+ System.err.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
fileOutput = true
srcVertexId = args(0).toLong
@@ -123,26 +125,29 @@ object GSASingleSourceShortestPaths {
outputPath = args(2)
maxIterations = 3
} else {
- System.out.println("Executing Single Source Shortest Paths example "
- + "with default parameters and built-in default data.")
+ System.out.println(
+ "Executing Single Source Shortest Paths example "
+ + "with default parameters and built-in default data.")
System.out.println(" Provide parameters to read input data from files.")
System.out.println(" See the documentation for the correct format of input files.")
- System.out.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ System.out.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
true
}
private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, Double]] = {
if (fileOutput) {
- env.readCsvFile[(Long, Long, Double)](edgesInputPath,
- lineDelimiter = "\n",
- fieldDelimiter = "\t")
+ env
+ .readCsvFile[(Long, Long, Double)](
+ edgesInputPath,
+ lineDelimiter = "\n",
+ fieldDelimiter = "\t")
.map(new Tuple3ToEdgeMap[Long, Double]())
} else {
- val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES map {
- case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long],
- z.asInstanceOf[Double])
+ val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES.map {
+ case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long], z.asInstanceOf[Double])
}
env.fromCollection(edgeData).map(new Tuple3ToEdgeMap[Long, Double]())
}
diff --git a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala
index f3b10f7d6f9..85ca764a855 100644
--- a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala
+++ b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala
@@ -15,29 +15,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.examples
import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.scala._
+import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.graph.examples.data.SingleSourceShortestPathsData
import org.apache.flink.graph.scala._
import org.apache.flink.graph.scala.utils.Tuple3ToEdgeMap
import org.apache.flink.graph.spargel.{GatherFunction, MessageIterator, ScatterFunction}
-import org.apache.flink.graph.{Edge, Vertex}
import scala.collection.JavaConversions._
/**
* This example shows how to use Gelly's scatter-gather iterations.
- *
- * It is an implementation of the Single-Source-Shortest-Paths algorithm.
*
- * The input file is a plain text file and must be formatted as follows:
- * Edges are represented by tuples of srcVertexId, trgVertexId, distance which are
- * separated by tabs. Edges themselves are separated by newlines.
- * For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges,
- * edge 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
+ * It is an implementation of the Single-Source-Shortest-Paths algorithm.
+ *
+ * The input file is a plain text file and must be formatted as follows: Edges are represented by
+ * tuples of srcVertexId, trgVertexId, distance which are separated by tabs. Edges themselves are
+ * separated by newlines. For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges, edge
+ * 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
*
* If no parameters are provided, the program is run with default data from
* [[SingleSourceShortestPathsData]]
@@ -53,8 +51,10 @@ object SingleSourceShortestPaths {
val graph = Graph.fromDataSet[Long, Double, Double](edges, new InitVertices(srcVertexId), env)
// Execute the scatter-gather iteration
- val result = graph.runScatterGatherIteration(new MinDistanceMessenger,
- new VertexDistanceUpdater, maxIterations)
+ val result = graph.runScatterGatherIteration(
+ new MinDistanceMessenger,
+ new VertexDistanceUpdater,
+ maxIterations)
// Extract the vertices as the result
val singleSourceShortestPaths = result.getVertices
@@ -72,7 +72,7 @@ object SingleSourceShortestPaths {
// Single Source Shortest Path UDFs
// --------------------------------------------------------------------------------------------
- private final class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
+ final private class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
override def map(id: Long) = {
if (id.equals(srcId)) {
@@ -84,11 +84,10 @@ object SingleSourceShortestPaths {
}
/**
- * Distributes the minimum distance associated with a given vertex among all
- * the target vertices summed up with the edge's value.
+ * Distributes the minimum distance associated with a given vertex among all the target vertices
+ * summed up with the edge's value.
*/
- private final class MinDistanceMessenger extends
- ScatterFunction[Long, Double, Double, Double] {
+ final private class MinDistanceMessenger extends ScatterFunction[Long, Double, Double, Double] {
override def sendMessages(vertex: Vertex[Long, Double]) {
if (vertex.getValue < Double.PositiveInfinity) {
@@ -100,10 +99,10 @@ object SingleSourceShortestPaths {
}
/**
- * Function that updates the value of a vertex by picking the minimum
- * distance from all incoming messages.
- */
- private final class VertexDistanceUpdater extends GatherFunction[Long, Double, Double] {
+ * Function that updates the value of a vertex by picking the minimum distance from all incoming
+ * messages.
+ */
+ final private class VertexDistanceUpdater extends GatherFunction[Long, Double, Double] {
override def updateVertex(vertex: Vertex[Long, Double], inMessages: MessageIterator[Double]) {
var minDistance = Double.MaxValue
@@ -130,10 +129,11 @@ object SingleSourceShortestPaths {
private var maxIterations = 5
private def parseParameters(args: Array[String]): Boolean = {
- if(args.length > 0) {
- if(args.length != 4) {
- System.err.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ if (args.length > 0) {
+ if (args.length != 4) {
+ System.err.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
fileOutput = true
srcVertexId = args(0).toLong
@@ -141,26 +141,29 @@ object SingleSourceShortestPaths {
outputPath = args(2)
maxIterations = 3
} else {
- System.out.println("Executing Single Source Shortest Paths example "
- + "with default parameters and built-in default data.")
+ System.out.println(
+ "Executing Single Source Shortest Paths example "
+ + "with default parameters and built-in default data.")
System.out.println(" Provide parameters to read input data from files.")
System.out.println(" See the documentation for the correct format of input files.")
- System.out.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ System.out.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
true
}
private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, Double]] = {
if (fileOutput) {
- env.readCsvFile[(Long, Long, Double)](edgesInputPath,
- lineDelimiter = "\n",
- fieldDelimiter = "\t")
+ env
+ .readCsvFile[(Long, Long, Double)](
+ edgesInputPath,
+ lineDelimiter = "\n",
+ fieldDelimiter = "\t")
.map(new Tuple3ToEdgeMap[Long, Double]())
} else {
- val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES map {
- case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long],
- z.asInstanceOf[Double])
+ val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES.map {
+ case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long], z.asInstanceOf[Double])
}
env.fromCollection(edgeData).map(new Tuple3ToEdgeMap[Long, Double]())
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala
index 70a5fdf181a..807e616855f 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
import org.apache.flink.api.java.tuple.Tuple2
@@ -26,9 +25,11 @@ abstract class EdgesFunction[K, EV, T] extends org.apache.flink.graph.EdgesFunct
def iterateEdges(edges: Iterable[(K, Edge[K, EV])], out: Collector[T])
- override def iterateEdges(edges: java.lang.Iterable[Tuple2[K, Edge[K, EV]]], out:
- Collector[T]): Unit = {
- val scalaIterable = scala.collection.JavaConversions.iterableAsScalaIterable(edges)
+ override def iterateEdges(
+ edges: java.lang.Iterable[Tuple2[K, Edge[K, EV]]],
+ out: Collector[T]): Unit = {
+ val scalaIterable = scala.collection.JavaConversions
+ .iterableAsScalaIterable(edges)
.map(jtuple => (jtuple.f0, jtuple.f1))
iterateEdges(scalaIterable, out)
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala
index 82589b6fbf9..0a67e600cea 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala
@@ -15,19 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.util.Collector
-abstract class EdgesFunctionWithVertexValue[K, VV, EV, T] extends org.apache.flink.graph
-.EdgesFunctionWithVertexValue[K, VV, EV, T] {
+abstract class EdgesFunctionWithVertexValue[K, VV, EV, T]
+ extends org.apache.flink.graph.EdgesFunctionWithVertexValue[K, VV, EV, T] {
@throws(classOf[Exception])
def iterateEdges(v: Vertex[K, VV], edges: Iterable[Edge[K, EV]], out: Collector[T])
- override def iterateEdges(v: Vertex[K, VV], edges: java.lang.Iterable[Edge[K, EV]], out:
- Collector[T]) = {
+ override def iterateEdges(
+ v: Vertex[K, VV],
+ edges: java.lang.Iterable[Edge[K, EV]],
+ out: Collector[T]) = {
iterateEdges(v, scala.collection.JavaConversions.iterableAsScalaIterable(edges), out)
}
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
index df7b8c10822..289d7c57e0b 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
@@ -15,163 +15,181 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
+import org.apache.flink.{graph => jg}
import org.apache.flink.api.common.functions.{FilterFunction, MapFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.{tuple => jtuple}
import org.apache.flink.api.scala._
import org.apache.flink.graph._
import org.apache.flink.graph.asm.translate.TranslateFunction
-import org.apache.flink.graph.gsa.{ApplyFunction, GSAConfiguration, SumFunction, GatherFunction => GSAGatherFunction}
+import org.apache.flink.graph.gsa.{ApplyFunction, GatherFunction => GSAGatherFunction, GSAConfiguration, SumFunction}
import org.apache.flink.graph.pregel.{ComputeFunction, MessageCombiner, VertexCentricConfiguration}
-import org.apache.flink.graph.spargel.{ScatterFunction, ScatterGatherConfiguration, GatherFunction => SpargelGatherFunction}
+import org.apache.flink.graph.spargel.{GatherFunction => SpargelGatherFunction, ScatterFunction, ScatterGatherConfiguration}
import org.apache.flink.graph.validation.GraphValidator
import org.apache.flink.types.{LongValue, NullValue}
import org.apache.flink.util.Preconditions
-import org.apache.flink.{graph => jg}
import _root_.scala.collection.JavaConverters._
import _root_.scala.reflect.ClassTag
object Graph {
- /**
- * Creates a Graph from a DataSet of vertices and a DataSet of edges.
- */
- def fromDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](vertices: DataSet[Vertex[K, VV]], edges: DataSet[Edge[K, EV]],
- env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ /** Creates a Graph from a DataSet of vertices and a DataSet of edges. */
+ def fromDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ vertices: DataSet[Vertex[K, VV]],
+ edges: DataSet[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
wrapGraph(jg.Graph.fromDataSet[K, VV, EV](vertices.javaSet, edges.javaSet, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of edges.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a DataSet of edges. Vertices are created automatically and their values
+ * are set to NullValue.
*/
- def fromDataSet[K: TypeInformation : ClassTag, EV: TypeInformation : ClassTag]
- (edges: DataSet[Edge[K, EV]], env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
+ def fromDataSet[K: TypeInformation: ClassTag, EV: TypeInformation: ClassTag](
+ edges: DataSet[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
wrapGraph(jg.Graph.fromDataSet[K, EV](edges.javaSet, env.getJavaEnv))
}
/**
- * Creates a graph from a DataSet of edges.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex ids.
+ * Creates a graph from a DataSet of edges. Vertices are created automatically and their values
+ * are set by applying the provided vertexValueInitializer map function to the vertex ids.
*/
- def fromDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](edges: DataSet[Edge[K, EV]],
- vertexValueInitializer: MapFunction[K, VV], env: ExecutionEnvironment): Graph[K, VV, EV] = {
- wrapGraph(jg.Graph.fromDataSet[K, VV, EV](edges.javaSet, vertexValueInitializer,
- env.getJavaEnv))
+ def fromDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ edges: DataSet[Edge[K, EV]],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ wrapGraph(
+ jg.Graph.fromDataSet[K, VV, EV](edges.javaSet, vertexValueInitializer, env.getJavaEnv))
}
- /**
- * Creates a Graph from a Seq of vertices and a Seq of edges.
- */
- def fromCollection[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](vertices: Seq[Vertex[K, VV]], edges: Seq[Edge[K, EV]], env:
- ExecutionEnvironment): Graph[K, VV, EV] = {
- wrapGraph(jg.Graph.fromCollection[K, VV, EV](vertices.asJavaCollection, edges
- .asJavaCollection, env.getJavaEnv))
+ /** Creates a Graph from a Seq of vertices and a Seq of edges. */
+ def fromCollection[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ vertices: Seq[Vertex[K, VV]],
+ edges: Seq[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ wrapGraph(jg.Graph
+ .fromCollection[K, VV, EV](vertices.asJavaCollection, edges.asJavaCollection, env.getJavaEnv))
}
/**
- * Creates a Graph from a Seq of edges.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a Seq of edges. Vertices are created automatically and their values are
+ * set to NullValue.
*/
- def fromCollection[K: TypeInformation : ClassTag, EV: TypeInformation : ClassTag]
- (edges: Seq[Edge[K, EV]], env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
+ def fromCollection[K: TypeInformation: ClassTag, EV: TypeInformation: ClassTag](
+ edges: Seq[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
wrapGraph(jg.Graph.fromCollection[K, EV](edges.asJavaCollection, env.getJavaEnv))
}
/**
- * Creates a graph from a Seq of edges.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex ids.
+ * Creates a graph from a Seq of edges. Vertices are created automatically and their values are
+ * set by applying the provided vertexValueInitializer map function to the vertex ids.
*/
- def fromCollection[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](edges: Seq[Edge[K, EV]], vertexValueInitializer: MapFunction[K, VV],
- env: ExecutionEnvironment): Graph[K, VV, EV] = {
- wrapGraph(jg.Graph.fromCollection[K, VV, EV](edges.asJavaCollection, vertexValueInitializer,
- env.getJavaEnv))
+ def fromCollection[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ edges: Seq[Edge[K, EV]],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ wrapGraph(
+ jg.Graph
+ .fromCollection[K, VV, EV](edges.asJavaCollection, vertexValueInitializer, env.getJavaEnv))
}
/**
- * Creates a graph from DataSets of tuples for vertices and for edges.
- * The first field of the Tuple2 vertex object will become the vertex ID
- * and the second field will become the vertex value.
- * The first field of the Tuple3 object for edges will become the source ID,
- * the second field will become the target ID, and the third field will become
- * the edge value.
+ * Creates a graph from DataSets of tuples for vertices and for edges. The first field of the
+ * Tuple2 vertex object will become the vertex ID and the second field will become the vertex
+ * value. The first field of the Tuple3 object for edges will become the source ID, the second
+ * field will become the target ID, and the third field will become the edge value.
*/
- def fromTupleDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](vertices: DataSet[(K, VV)], edges: DataSet[(K, K, EV)],
- env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ def fromTupleDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ vertices: DataSet[(K, VV)],
+ edges: DataSet[(K, K, EV)],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
val javaTupleVertices = vertices.map((v: (K, VV)) => new jtuple.Tuple2(v._1, v._2)).javaSet
val javaTupleEdges = edges.map((v: (K, K, EV)) => new jtuple.Tuple3(v._1, v._2, v._3)).javaSet
- wrapGraph(jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleVertices, javaTupleEdges,
- env.getJavaEnv))
+ wrapGraph(
+ jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleVertices, javaTupleEdges, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuples representing the edges.
- * The first field of the Tuple3 object for edges will become the source ID,
- * the second field will become the target ID, and the third field will become
- * the edge value.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a DataSet of Tuples representing the edges. The first field of the Tuple3
+ * object for edges will become the source ID, the second field will become the target ID, and the
+ * third field will become the edge value. Vertices are created automatically and their values are
+ * set to NullValue.
*/
- def fromTupleDataSet[K: TypeInformation : ClassTag, EV: TypeInformation : ClassTag]
- (edges: DataSet[(K, K, EV)], env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
+ def fromTupleDataSet[K: TypeInformation: ClassTag, EV: TypeInformation: ClassTag](
+ edges: DataSet[(K, K, EV)],
+ env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
val javaTupleEdges = edges.map((v: (K, K, EV)) => new jtuple.Tuple3(v._1, v._2, v._3)).javaSet
wrapGraph(jg.Graph.fromTupleDataSet[K, EV](javaTupleEdges, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuples representing the edges.
- * The first field of the Tuple3 object for edges will become the source ID,
- * the second field will become the target ID, and the third field will become
- * the edge value.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex ids.
+ * Creates a Graph from a DataSet of Tuples representing the edges. The first field of the Tuple3
+ * object for edges will become the source ID, the second field will become the target ID, and the
+ * third field will become the edge value. Vertices are created automatically and their values are
+ * set by applying the provided vertexValueInitializer map function to the vertex ids.
*/
- def fromTupleDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](edges: DataSet[(K, K, EV)],
- vertexValueInitializer: MapFunction[K, VV], env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ def fromTupleDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ edges: DataSet[(K, K, EV)],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
val javaTupleEdges = edges.map((v: (K, K, EV)) => new jtuple.Tuple3(v._1, v._2, v._3)).javaSet
- wrapGraph(jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleEdges, vertexValueInitializer,
- env.getJavaEnv))
+ wrapGraph(
+ jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleEdges, vertexValueInitializer, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuple2's representing the edges.
- * The first field of the Tuple2 object for edges will become the source ID,
- * the second field will become the target ID. The edge value will be set to NullValue.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a DataSet of Tuple2's representing the edges. The first field of the
+ * Tuple2 object for edges will become the source ID, the second field will become the target ID.
+ * The edge value will be set to NullValue. Vertices are created automatically and their values
+ * are set to NullValue.
*/
- def fromTuple2DataSet[K: TypeInformation : ClassTag](edges: DataSet[(K, K)],
- env: ExecutionEnvironment): Graph[K, NullValue, NullValue] = {
+ def fromTuple2DataSet[K: TypeInformation: ClassTag](
+ edges: DataSet[(K, K)],
+ env: ExecutionEnvironment): Graph[K, NullValue, NullValue] = {
val javaTupleEdges = edges.map((v: (K, K)) => new jtuple.Tuple2(v._1, v._2)).javaSet
wrapGraph(jg.Graph.fromTuple2DataSet[K](javaTupleEdges, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuple2's representing the edges.
- * The first field of the Tuple2 object for edges will become the source ID,
- * the second field will become the target ID. The edge value will be set to NullValue.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex IDs.
+ * Creates a Graph from a DataSet of Tuple2's representing the edges. The first field of the
+ * Tuple2 object for edges will become the source ID, the second field will become the target ID.
+ * The edge value will be set to NullValue. Vertices are created automatically and their values
+ * are set by applying the provided vertexValueInitializer map function to the vertex IDs.
*/
- def fromTuple2DataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag]
- (edges: DataSet[(K, K)], vertexValueInitializer: MapFunction[K, VV],
- env: ExecutionEnvironment): Graph[K, VV, NullValue] = {
+ def fromTuple2DataSet[K: TypeInformation: ClassTag, VV: TypeInformation: ClassTag](
+ edges: DataSet[(K, K)],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, NullValue] = {
val javaTupleEdges = edges.map((v: (K, K)) => new jtuple.Tuple2(v._1, v._2)).javaSet
- wrapGraph(jg.Graph.fromTuple2DataSet[K, VV](javaTupleEdges, vertexValueInitializer,
- env.getJavaEnv))
+ wrapGraph(
+ jg.Graph.fromTuple2DataSet[K, VV](javaTupleEdges, vertexValueInitializer, env.getJavaEnv))
}
- /** Creates a Graph from a CSV file of edges.
+ /**
+ * Creates a Graph from a CSV file of edges.
*
* The edge value is read from the CSV file if [[EV]] is not of type [[NullValue]]. Otherwise the
* edge value is set to [[NullValue]].
@@ -179,50 +197,59 @@ object Graph {
* If the vertex value type [[VV]] is specified (unequal [[NullValue]]), then the vertex values
* are read from the file specified by pathVertices. If the path has not been specified then the
* vertexValueInitializer is used to initialize the vertex values of the vertices extracted from
- * the set of edges. If the vertexValueInitializer has not been set either, then the method
- * fails.
+ * the set of edges. If the vertexValueInitializer has not been set either, then the method fails.
*
- * @param env The Execution Environment.
- * @param pathEdges The file path containing the edges.
- * @param pathVertices The file path containing the vertices.
- * @param lineDelimiterVertices The string that separates lines in the vertices file. It defaults
- * to newline.
- * @param fieldDelimiterVertices The string that separates vertex Ids from vertex values in the
- * vertices file.
- * @param quoteCharacterVertices The character to use for quoted String parsing in the vertices
- * file. Disabled by default.
- * @param ignoreFirstLineVertices Whether the first line in the vertices file should be ignored.
- * @param ignoreCommentsVertices Lines that start with the given String in the vertices file
- * are ignored, disabled by default.
- * @param lenientVertices Whether the parser should silently ignore malformed lines in the
- * vertices file.
- * @param includedFieldsVertices The fields in the vertices file that should be read. By default
- * all fields are read.
- * @param lineDelimiterEdges The string that separates lines in the edges file. It defaults to
- * newline.
- * @param fieldDelimiterEdges The string that separates fields in the edges file.
- * @param quoteCharacterEdges The character to use for quoted String parsing in the edges file.
- * Disabled by default.
- * @param ignoreFirstLineEdges Whether the first line in the vertices file should be ignored.
- * @param ignoreCommentsEdges Lines that start with the given String in the edges file are
- * ignored, disabled by default.
- * @param lenientEdges Whether the parser should silently ignore malformed lines in the edges
- * file.
- * @param includedFieldsEdges The fields in the edges file that should be read. By default all
- * fields are read.
- * @param vertexValueInitializer If no vertex values are provided, this mapper can be used to
- * initialize them, by applying a map transformation on the vertex
- * IDs.
- * @tparam K Vertex key type
- * @tparam VV Vertex value type
- * @tparam EV Edge value type
- * @return Graph with vertices and edges read from the given files.
- */
- // This method exceeds the max allowed number of parameters -->
+ * @param env
+ * The Execution Environment.
+ * @param pathEdges
+ * The file path containing the edges.
+ * @param pathVertices
+ * The file path containing the vertices.
+ * @param lineDelimiterVertices
+ * The string that separates lines in the vertices file. It defaults to newline.
+ * @param fieldDelimiterVertices
+ * The string that separates vertex Ids from vertex values in the vertices file.
+ * @param quoteCharacterVertices
+ * The character to use for quoted String parsing in the vertices file. Disabled by default.
+ * @param ignoreFirstLineVertices
+ * Whether the first line in the vertices file should be ignored.
+ * @param ignoreCommentsVertices
+ * Lines that start with the given String in the vertices file are ignored, disabled by default.
+ * @param lenientVertices
+ * Whether the parser should silently ignore malformed lines in the vertices file.
+ * @param includedFieldsVertices
+ * The fields in the vertices file that should be read. By default all fields are read.
+ * @param lineDelimiterEdges
+ * The string that separates lines in the edges file. It defaults to newline.
+ * @param fieldDelimiterEdges
+ * The string that separates fields in the edges file.
+ * @param quoteCharacterEdges
+ * The character to use for quoted String parsing in the edges file. Disabled by default.
+ * @param ignoreFirstLineEdges
+ * Whether the first line in the vertices file should be ignored.
+ * @param ignoreCommentsEdges
+ * Lines that start with the given String in the edges file are ignored, disabled by default.
+ * @param lenientEdges
+ * Whether the parser should silently ignore malformed lines in the edges file.
+ * @param includedFieldsEdges
+ * The fields in the edges file that should be read. By default all fields are read.
+ * @param vertexValueInitializer
+ * If no vertex values are provided, this mapper can be used to initialize them, by applying a
+ * map transformation on the vertex IDs.
+ * @tparam K
+ * Vertex key type
+ * @tparam VV
+ * Vertex value type
+ * @tparam EV
+ * Edge value type
+ * @return
+ * Graph with vertices and edges read from the given files.
+ */
+ // This method exceeds the max allowed number of parameters -->
def fromCsvReader[
- K: TypeInformation : ClassTag,
- VV: TypeInformation : ClassTag,
- EV: TypeInformation : ClassTag](
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
env: ExecutionEnvironment,
pathEdges: String,
pathVertices: String = null,
@@ -240,8 +267,7 @@ object Graph {
ignoreCommentsEdges: String = null,
lenientEdges: Boolean = false,
includedFieldsEdges: Array[Int] = null,
- vertexValueInitializer: MapFunction[K, VV] = null)
- : Graph[K, VV, EV] = {
+ vertexValueInitializer: MapFunction[K, VV] = null): Graph[K, VV, EV] = {
Preconditions.checkNotNull(pathEdges)
@@ -249,15 +275,16 @@ object Graph {
val vvClassTag = implicitly[ClassTag[VV]]
val edges = if (evClassTag.runtimeClass.equals(classOf[NullValue])) {
- env.readCsvFile[(K, K)](
- pathEdges,
- lineDelimiterEdges,
- fieldDelimiterEdges,
- quoteCharacterEdges,
- ignoreFirstLineEdges,
- ignoreCommentsEdges,
- lenientEdges,
- includedFieldsEdges)
+ env
+ .readCsvFile[(K, K)](
+ pathEdges,
+ lineDelimiterEdges,
+ fieldDelimiterEdges,
+ quoteCharacterEdges,
+ ignoreFirstLineEdges,
+ ignoreCommentsEdges,
+ lenientEdges,
+ includedFieldsEdges)
.map((edge: (K, K)) => (edge._1, edge._2, NullValue.getInstance))
.asInstanceOf[DataSet[(K, K, EV)]]
} else {
@@ -276,16 +303,24 @@ object Graph {
fromTupleDataSet[K, EV](edges, env).asInstanceOf[Graph[K, VV, EV]]
} else {
if (pathVertices != null) {
- val vertices = env.readCsvFile[(K, VV)](pathVertices, lineDelimiterVertices,
- fieldDelimiterVertices, quoteCharacterVertices, ignoreFirstLineVertices,
- ignoreCommentsVertices, lenientVertices, includedFieldsVertices)
+ val vertices = env.readCsvFile[(K, VV)](
+ pathVertices,
+ lineDelimiterVertices,
+ fieldDelimiterVertices,
+ quoteCharacterVertices,
+ ignoreFirstLineVertices,
+ ignoreCommentsVertices,
+ lenientVertices,
+ includedFieldsVertices
+ )
fromTupleDataSet[K, VV, EV](vertices, edges, env)
} else if (vertexValueInitializer != null) {
fromTupleDataSet[K, VV, EV](edges, vertexValueInitializer, env)
} else {
- throw new IllegalArgumentException("Path vertices path and vertex value initialzier must" +
- "not be null if the vertex value type is not NullValue.")
+ throw new IllegalArgumentException(
+ "Path vertices path and vertex value initialzier must" +
+ "not be null if the vertex value type is not NullValue.")
}
}
}
@@ -294,19 +329,26 @@ object Graph {
/**
* Represents a graph consisting of [[Edge]] edges and [[Vertex]] vertices.
*
- * @param jgraph the underlying java api Graph.
- * @tparam K the key type for vertex and edge identifiers
- * @tparam VV the value type for vertices
- * @tparam EV the value type for edges
- * @see org.apache.flink.graph.Edge
- * @see org.apache.flink.graph.Vertex
+ * @param jgraph
+ * the underlying java api Graph.
+ * @tparam K
+ * the key type for vertex and edge identifiers
+ * @tparam VV
+ * the value type for vertices
+ * @tparam EV
+ * the value type for edges
+ * @see
+ * org.apache.flink.graph.Edge
+ * @see
+ * org.apache.flink.graph.Vertex
*/
-final class Graph[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
-TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
+final class Graph[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](jgraph: jg.Graph[K, VV, EV]) {
private[flink] def getWrappedGraph = jgraph
-
private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
if (jgraph.getContext.getConfig.isClosureCleanerEnabled) {
ClosureCleaner.clean(f, checkSerializable, jgraph.getContext.getConfig.getClosureCleanerLevel)
@@ -315,36 +357,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
f
}
- /**
- * @return the vertex DataSet.
- */
+ /** @return the vertex DataSet. */
def getVertices = wrap(jgraph.getVertices)
- /**
- * @return the edge DataSet.
- */
+ /** @return the edge DataSet. */
def getEdges = wrap(jgraph.getEdges)
- /**
- * @return the vertex DataSet as Tuple2.
- */
+ /** @return the vertex DataSet as Tuple2. */
def getVerticesAsTuple2(): DataSet[(K, VV)] = {
- wrap(jgraph.getVerticesAsTuple2).map(
- (v: jtuple.Tuple2[K, VV]) => (v.f0, v.f1))
+ wrap(jgraph.getVerticesAsTuple2).map((v: jtuple.Tuple2[K, VV]) => (v.f0, v.f1))
}
- /**
- * @return the edge DataSet as Tuple3.
- */
+ /** @return the edge DataSet as Tuple3. */
def getEdgesAsTuple3(): DataSet[(K, K, EV)] = {
- wrap(jgraph.getEdgesAsTuple3).map(
- (e: jtuple.Tuple3[K, K, EV]) =>
- (e.f0, e.f1, e.f2))
+ wrap(jgraph.getEdgesAsTuple3).map((e: jtuple.Tuple3[K, K, EV]) => (e.f0, e.f1, e.f2))
}
/**
- * @return a DataSet of Triplets,
- * consisting of (srcVertexId, trgVertexId, srcVertexValue, trgVertexValue, edgeValue)
+ * @return
+ * a DataSet of Triplets, consisting of (srcVertexId, trgVertexId, srcVertexValue,
+ * trgVertexValue, edgeValue)
*/
def getTriplets(): DataSet[Triplet[K, VV, EV]] = {
wrap(jgraph.getTriplets())
@@ -353,24 +385,29 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Apply a function to the attribute of each vertex in the graph.
*
- * @param mapper the map function to apply.
- * @return a new graph
+ * @param mapper
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapVertices[NV: TypeInformation : ClassTag](mapper: MapFunction[Vertex[K, VV], NV]):
- Graph[K, NV, EV] = {
- new Graph[K, NV, EV](jgraph.mapVertices[NV](
- mapper,
- createTypeInformation[Vertex[K, NV]]
- ))
+ def mapVertices[NV: TypeInformation: ClassTag](
+ mapper: MapFunction[Vertex[K, VV], NV]): Graph[K, NV, EV] = {
+ new Graph[K, NV, EV](
+ jgraph.mapVertices[NV](
+ mapper,
+ createTypeInformation[Vertex[K, NV]]
+ ))
}
/**
* Apply a function to the attribute of each vertex in the graph.
*
- * @param fun the map function to apply.
- * @return a new graph
+ * @param fun
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapVertices[NV: TypeInformation : ClassTag](fun: Vertex[K, VV] => NV): Graph[K, NV, EV] = {
+ def mapVertices[NV: TypeInformation: ClassTag](fun: Vertex[K, VV] => NV): Graph[K, NV, EV] = {
val mapper: MapFunction[Vertex[K, VV], NV] = new MapFunction[Vertex[K, VV], NV] {
val cleanFun = clean(fun)
@@ -382,24 +419,29 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Apply a function to the attribute of each edge in the graph.
*
- * @param mapper the map function to apply.
- * @return a new graph
+ * @param mapper
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapEdges[NV: TypeInformation : ClassTag](mapper: MapFunction[Edge[K, EV], NV]): Graph[K,
- VV, NV] = {
- new Graph[K, VV, NV](jgraph.mapEdges[NV](
- mapper,
- createTypeInformation[Edge[K, NV]]
- ))
+ def mapEdges[NV: TypeInformation: ClassTag](
+ mapper: MapFunction[Edge[K, EV], NV]): Graph[K, VV, NV] = {
+ new Graph[K, VV, NV](
+ jgraph.mapEdges[NV](
+ mapper,
+ createTypeInformation[Edge[K, NV]]
+ ))
}
/**
* Apply a function to the attribute of each edge in the graph.
*
- * @param fun the map function to apply.
- * @return a new graph
+ * @param fun
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapEdges[NV: TypeInformation : ClassTag](fun: Edge[K, EV] => NV): Graph[K, VV, NV] = {
+ def mapEdges[NV: TypeInformation: ClassTag](fun: Edge[K, EV] => NV): Graph[K, VV, NV] = {
val mapper: MapFunction[Edge[K, EV], NV] = new MapFunction[Edge[K, EV], NV] {
val cleanFun = clean(fun)
@@ -411,22 +453,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Translate vertex and edge IDs using the given MapFunction.
*
- * @param translator implements conversion from K to NEW
- * @return graph with translated vertex and edge IDs
+ * @param translator
+ * implements conversion from K to NEW
+ * @return
+ * graph with translated vertex and edge IDs
*/
- def translateGraphIds[NEW: TypeInformation : ClassTag](translator: TranslateFunction[K, NEW]):
- Graph[NEW, VV, EV] = {
+ def translateGraphIds[NEW: TypeInformation: ClassTag](
+ translator: TranslateFunction[K, NEW]): Graph[NEW, VV, EV] = {
new Graph[NEW, VV, EV](jgraph.translateGraphIds(translator))
}
/**
* Translate vertex and edge IDs using the given function.
*
- * @param fun implements conversion from K to NEW
- * @return graph with translated vertex and edge IDs
+ * @param fun
+ * implements conversion from K to NEW
+ * @return
+ * graph with translated vertex and edge IDs
*/
- def translateGraphIds[NEW: TypeInformation : ClassTag](fun: (K, NEW) => NEW):
- Graph[NEW, VV, EV] = {
+ def translateGraphIds[NEW: TypeInformation: ClassTag](
+ fun: (K, NEW) => NEW): Graph[NEW, VV, EV] = {
val translator: TranslateFunction[K, NEW] = new TranslateFunction[K, NEW] {
val cleanFun = clean(fun)
@@ -439,22 +485,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Translate vertex values using the given MapFunction.
*
- * @param translator implements conversion from VV to NEW
- * @return graph with translated vertex values
+ * @param translator
+ * implements conversion from VV to NEW
+ * @return
+ * graph with translated vertex values
*/
- def translateVertexValues[NEW: TypeInformation : ClassTag](translator:
- TranslateFunction[VV, NEW]): Graph[K, NEW, EV] = {
+ def translateVertexValues[NEW: TypeInformation: ClassTag](
+ translator: TranslateFunction[VV, NEW]): Graph[K, NEW, EV] = {
new Graph[K, NEW, EV](jgraph.translateVertexValues(translator))
}
/**
* Translate vertex values using the given function.
*
- * @param fun implements conversion from VV to NEW
- * @return graph with translated vertex values
+ * @param fun
+ * implements conversion from VV to NEW
+ * @return
+ * graph with translated vertex values
*/
- def translateVertexValues[NEW: TypeInformation : ClassTag](fun: (VV, NEW) => NEW):
- Graph[K, NEW, EV] = {
+ def translateVertexValues[NEW: TypeInformation: ClassTag](
+ fun: (VV, NEW) => NEW): Graph[K, NEW, EV] = {
val translator: TranslateFunction[VV, NEW] = new TranslateFunction[VV, NEW] {
val cleanFun = clean(fun)
@@ -467,22 +517,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Translate edge values using the given MapFunction.
*
- * @param translator implements conversion from EV to NEW
- * @return graph with translated edge values
+ * @param translator
+ * implements conversion from EV to NEW
+ * @return
+ * graph with translated edge values
*/
- def translateEdgeValues[NEW: TypeInformation : ClassTag](translator: TranslateFunction[EV, NEW]):
- Graph[K, VV, NEW] = {
+ def translateEdgeValues[NEW: TypeInformation: ClassTag](
+ translator: TranslateFunction[EV, NEW]): Graph[K, VV, NEW] = {
new Graph[K, VV, NEW](jgraph.translateEdgeValues(translator))
}
/**
* Translate edge values using the given function.
*
- * @param fun implements conversion from EV to NEW
- * @return graph with translated edge values
+ * @param fun
+ * implements conversion from EV to NEW
+ * @return
+ * graph with translated edge values
*/
- def translateEdgeValues[NEW: TypeInformation : ClassTag](fun: (EV, NEW) => NEW):
- Graph[K, VV, NEW] = {
+ def translateEdgeValues[NEW: TypeInformation: ClassTag](
+ fun: (EV, NEW) => NEW): Graph[K, VV, NEW] = {
val translator: TranslateFunction[EV, NEW] = new TranslateFunction[EV, NEW] {
val cleanFun = clean(fun)
@@ -493,44 +547,49 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies
- * a user-defined transformation on the values of the matched records.
- * The vertex ID and the first field of the Tuple2 DataSet are used as the join keys.
- *
- * @param inputDataSet the Tuple2 DataSet to join with.
- * The first field of the Tuple2 is used as the join key and the second field is passed
- * as a parameter to the transformation function.
- * @param vertexJoinFunction the transformation function to apply.
- * The first parameter is the current vertex value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @return a new Graph, where the vertex values have been updated according to the
- * result of the vertexJoinFunction.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- */
- def joinWithVertices[T: TypeInformation](inputDataSet: DataSet[(K, T)],
- vertexJoinFunction: VertexJoinFunction[VV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies a user-defined
+ * transformation on the values of the matched records. The vertex ID and the first field of the
+ * Tuple2 DataSet are used as the join keys.
+ *
+ * @param inputDataSet
+ * the Tuple2 DataSet to join with. The first field of the Tuple2 is used as the join key and
+ * the second field is passed as a parameter to the transformation function.
+ * @param vertexJoinFunction
+ * the transformation function to apply. The first parameter is the current vertex value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @return
+ * a new Graph, where the vertex values have been updated according to the result of the
+ * vertexJoinFunction.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ */
+ def joinWithVertices[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ vertexJoinFunction: VertexJoinFunction[VV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithVertices[T](javaTupleSet, vertexJoinFunction))
}
/**
- * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies
- * a user-defined transformation on the values of the matched records.
- * The vertex ID and the first field of the Tuple2 DataSet are used as the join keys.
- *
- * @param inputDataSet the Tuple2 DataSet to join with.
- * The first field of the Tuple2 is used as the join key and the second field is passed
- * as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current vertex value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @return a new Graph, where the vertex values have been updated according to the
- * result of the vertexJoinFunction.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- */
- def joinWithVertices[T: TypeInformation](inputDataSet: DataSet[(K, T)], fun: (VV, T) => VV):
- Graph[K, VV, EV] = {
+ * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies a user-defined
+ * transformation on the values of the matched records. The vertex ID and the first field of the
+ * Tuple2 DataSet are used as the join keys.
+ *
+ * @param inputDataSet
+ * the Tuple2 DataSet to join with. The first field of the Tuple2 is used as the join key and
+ * the second field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current vertex value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @return
+ * a new Graph, where the vertex values have been updated according to the result of the
+ * vertexJoinFunction.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ */
+ def joinWithVertices[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ fun: (VV, T) => VV): Graph[K, VV, EV] = {
val newVertexJoin = new VertexJoinFunction[VV, T]() {
val cleanFun = clean(fun)
@@ -538,50 +597,55 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
cleanFun(vertexValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithVertices[T](javaTupleSet, newVertexJoin))
}
/**
- * Joins the edge DataSet with an input DataSet on the composite key of both
- * source and target IDs and applies a user-defined transformation on the values
- * of the matched records. The first two fields of the input DataSet are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first two fields of the Tuple3 are used as the composite join key
- * and the third field is passed as a parameter to the transformation function.
- * @param edgeJoinFunction the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple3 from the input DataSet.
- * @tparam T the type of the third field of the input Tuple3 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdges[T: TypeInformation](inputDataSet: DataSet[(K, K, T)],
- edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
+ * Joins the edge DataSet with an input DataSet on the composite key of both source and target IDs
+ * and applies a user-defined transformation on the values of the matched records. The first two
+ * fields of the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first two fields of the Tuple3 are used as the composite join
+ * key and the third field is passed as a parameter to the transformation function.
+ * @param edgeJoinFunction
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple3 from the input DataSet.
+ * @tparam T
+ * the type of the third field of the input Tuple3 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdges[T: TypeInformation](
+ inputDataSet: DataSet[(K, K, T)],
+ edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet =
+ inputDataSet.map((i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
wrapGraph(jgraph.joinWithEdges[T](javaTupleSet, edgeJoinFunction))
}
/**
- * Joins the edge DataSet with an input DataSet on the composite key of both
- * source and target IDs and applies a user-defined transformation on the values
- * of the matched records. The first two fields of the input DataSet are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first two fields of the Tuple3 are used as the composite join key
- * and the third field is passed as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple3 from the input DataSet.
- * @tparam T the type of the third field of the input Tuple3 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdges[T: TypeInformation](inputDataSet: DataSet[(K, K, T)], fun: (EV, T) => EV):
- Graph[K, VV, EV] = {
+ * Joins the edge DataSet with an input DataSet on the composite key of both source and target IDs
+ * and applies a user-defined transformation on the values of the matched records. The first two
+ * fields of the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first two fields of the Tuple3 are used as the composite join
+ * key and the third field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple3 from the input DataSet.
+ * @tparam T
+ * the type of the third field of the input Tuple3 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdges[T: TypeInformation](
+ inputDataSet: DataSet[(K, K, T)],
+ fun: (EV, T) => EV): Graph[K, VV, EV] = {
val newEdgeJoin = new EdgeJoinFunction[EV, T]() {
val cleanFun = clean(fun)
@@ -589,52 +653,55 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
cleanFun(edgeValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
+ val javaTupleSet =
+ inputDataSet.map((i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
wrapGraph(jgraph.joinWithEdges[T](javaTupleSet, newEdgeJoin))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The source ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param edgeJoinFunction the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnSource[T: TypeInformation](inputDataSet: DataSet[(K, T)],
- edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ * on the values of the matched records. The source ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param edgeJoinFunction
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnSource[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnSource[T](javaTupleSet, edgeJoinFunction))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The source ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnSource[T: TypeInformation](inputDataSet: DataSet[(K, T)], fun: (EV, T) =>
- EV): Graph[K, VV, EV] = {
+ * on the values of the matched records. The source ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnSource[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ fun: (EV, T) => EV): Graph[K, VV, EV] = {
val newEdgeJoin = new EdgeJoinFunction[EV, T]() {
val cleanFun = clean(fun)
@@ -642,87 +709,94 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
cleanFun(edgeValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnSource[T](javaTupleSet, newEdgeJoin))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The target ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param edgeJoinFunction the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnTarget[T: TypeInformation](inputDataSet: DataSet[(K, T)],
- edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ * on the values of the matched records. The target ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param edgeJoinFunction
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnTarget[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnTarget[T](javaTupleSet, edgeJoinFunction))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The target ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnTarget[T: TypeInformation](inputDataSet: DataSet[(K, T)], fun: (EV, T) =>
- EV): Graph[K, VV, EV] = {
+ * on the values of the matched records. The target ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnTarget[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ fun: (EV, T) => EV): Graph[K, VV, EV] = {
val newEdgeJoin = new EdgeJoinFunction[EV, T]() {
val cleanFun = clean(fun)
- override def edgeJoin(edgeValue: EV, inputValue:T): EV = {
+ override def edgeJoin(edgeValue: EV, inputValue: T): EV = {
cleanFun(edgeValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnTarget[T](javaTupleSet, newEdgeJoin))
}
/**
- * Apply filtering functions to the graph and return a sub-graph that
- * satisfies the predicates for both vertices and edges.
+ * Apply filtering functions to the graph and return a sub-graph that satisfies the predicates for
+ * both vertices and edges.
*
- * @param vertexFilter the filter function for vertices.
- * @param edgeFilter the filter function for edges.
- * @return the resulting sub-graph.
- */
- def subgraph(vertexFilter: FilterFunction[Vertex[K, VV]], edgeFilter: FilterFunction[Edge[K,
- EV]]) = {
+ * @param vertexFilter
+ * the filter function for vertices.
+ * @param edgeFilter
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
+ */
+ def subgraph(
+ vertexFilter: FilterFunction[Vertex[K, VV]],
+ edgeFilter: FilterFunction[Edge[K, EV]]) = {
wrapGraph(jgraph.subgraph(vertexFilter, edgeFilter))
}
/**
- * Apply filtering functions to the graph and return a sub-graph that
- * satisfies the predicates for both vertices and edges.
+ * Apply filtering functions to the graph and return a sub-graph that satisfies the predicates for
+ * both vertices and edges.
*
- * @param vertexFilterFun the filter function for vertices.
- * @param edgeFilterFun the filter function for edges.
- * @return the resulting sub-graph.
- */
- def subgraph(vertexFilterFun: Vertex[K, VV] => Boolean, edgeFilterFun: Edge[K, EV] =>
- Boolean) = {
+ * @param vertexFilterFun
+ * the filter function for vertices.
+ * @param edgeFilterFun
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
+ */
+ def subgraph(vertexFilterFun: Vertex[K, VV] => Boolean, edgeFilterFun: Edge[K, EV] => Boolean) = {
val vertexFilter = new FilterFunction[Vertex[K, VV]] {
val cleanVertexFun = clean(vertexFilterFun)
@@ -739,22 +813,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the vertices.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the vertices.
*
- * @param vertexFilter the filter function for vertices.
- * @return the resulting sub-graph.
+ * @param vertexFilter
+ * the filter function for vertices.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnVertices(vertexFilter: FilterFunction[Vertex[K, VV]]) = {
wrapGraph(jgraph.filterOnVertices(vertexFilter))
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the vertices.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the vertices.
*
- * @param vertexFilterFun the filter function for vertices.
- * @return the resulting sub-graph.
+ * @param vertexFilterFun
+ * the filter function for vertices.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnVertices(vertexFilterFun: Vertex[K, VV] => Boolean) = {
val vertexFilter = new FilterFunction[Vertex[K, VV]] {
@@ -767,22 +845,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the edges.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the edges.
*
- * @param edgeFilter the filter function for edges.
- * @return the resulting sub-graph.
+ * @param edgeFilter
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnEdges(edgeFilter: FilterFunction[Edge[K, EV]]) = {
wrapGraph(jgraph.filterOnEdges(edgeFilter))
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the edges.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the edges.
*
- * @param edgeFilterFun the filter function for edges.
- * @return the resulting sub-graph.
+ * @param edgeFilterFun
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnEdges(edgeFilterFun: Edge[K, EV] => Boolean) = {
val edgeFilter = new FilterFunction[Edge[K, EV]] {
@@ -797,7 +879,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Return the in-degree of all vertices in the graph
*
- * @return A DataSet of Tuple2<vertexId, inDegree>
+ * @return
+ * A DataSet of Tuple2<vertexId, inDegree>
*/
def inDegrees(): DataSet[(K, LongValue)] = {
wrap(jgraph.inDegrees).map((i: jtuple.Tuple2[K, LongValue]) => (i.f0, i.f1))
@@ -806,7 +889,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Return the out-degree of all vertices in the graph
*
- * @return A DataSet of Tuple2<vertexId, outDegree>
+ * @return
+ * A DataSet of Tuple2<vertexId, outDegree>
*/
def outDegrees(): DataSet[(K, LongValue)] = {
wrap(jgraph.outDegrees).map((i: jtuple.Tuple2[K, LongValue]) => (i.f0, i.f1))
@@ -815,7 +899,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Return the degree of all vertices in the graph
*
- * @return A DataSet of Tuple2<vertexId, degree>
+ * @return
+ * A DataSet of Tuple2<vertexId, degree>
*/
def getDegrees(): DataSet[(K, LongValue)] = {
wrap(jgraph.getDegrees).map((i: jtuple.Tuple2[K, LongValue]) => (i.f0, i.f1))
@@ -824,7 +909,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* This operation adds all inverse-direction edges to the graph.
*
- * @return the undirected graph.
+ * @return
+ * the undirected graph.
*/
def getUndirected(): Graph[K, VV, EV] = {
new Graph(jgraph.getUndirected)
@@ -833,7 +919,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Reverse the direction of the edges in the graph
*
- * @return a new graph with all edges reversed
+ * @return
+ * a new graph with all edges reversed
* @throws UnsupportedOperationException
*/
def reverse(): Graph[K, VV, EV] = {
@@ -841,113 +928,121 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Compute an aggregate over the edges of each vertex. The function applied
- * on the edges has access to the vertex value.
+ * Compute an aggregate over the edges of each vertex. The function applied on the edges has
+ * access to the vertex value.
*
- * @param edgesFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnEdges[T: TypeInformation : ClassTag](edgesFunction:
- EdgesFunctionWithVertexValue[K, VV, EV,
- T], direction: EdgeDirection):
- DataSet[T] = {
+ * @param edgesFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnEdges[T: TypeInformation: ClassTag](
+ edgesFunction: EdgesFunctionWithVertexValue[K, VV, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnEdges(edgesFunction, direction, createTypeInformation[T]))
}
/**
- * Compute an aggregate over the edges of each vertex. The function applied
- * on the edges has access to the vertex value.
+ * Compute an aggregate over the edges of each vertex. The function applied on the edges has
+ * access to the vertex value.
*
- * @param edgesFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnEdges[T: TypeInformation : ClassTag](edgesFunction: EdgesFunction[K, EV, T],
- direction: EdgeDirection): DataSet[T] = {
+ * @param edgesFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnEdges[T: TypeInformation: ClassTag](
+ edgesFunction: EdgesFunction[K, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnEdges(edgesFunction, direction, createTypeInformation[T]))
}
/**
- * Compute an aggregate over the neighbors (edges and vertices) of each
- * vertex. The function applied on the neighbors has access to the vertex
- * value.
+ * Compute an aggregate over the neighbors (edges and vertices) of each vertex. The function
+ * applied on the neighbors has access to the vertex value.
*
- * @param neighborsFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnNeighbors[T: TypeInformation : ClassTag](neighborsFunction:
- NeighborsFunctionWithVertexValue[K,
- VV, EV, T], direction:
- EdgeDirection): DataSet[T] = {
+ * @param neighborsFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnNeighbors[T: TypeInformation: ClassTag](
+ neighborsFunction: NeighborsFunctionWithVertexValue[K, VV, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnNeighbors(neighborsFunction, direction, createTypeInformation[T]))
}
/**
- * Compute an aggregate over the neighbors (edges and vertices) of each
- * vertex.
+ * Compute an aggregate over the neighbors (edges and vertices) of each vertex.
*
- * @param neighborsFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnNeighbors[T: TypeInformation : ClassTag](neighborsFunction:
- NeighborsFunction[K, VV, EV, T],
- direction: EdgeDirection):
- DataSet[T] = {
+ * @param neighborsFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnNeighbors[T: TypeInformation: ClassTag](
+ neighborsFunction: NeighborsFunction[K, VV, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnNeighbors(neighborsFunction, direction, createTypeInformation[T]))
}
- /**
- * @return a long integer representing the number of vertices
- */
+ /** @return a long integer representing the number of vertices */
def numberOfVertices(): Long = {
jgraph.numberOfVertices()
}
- /**
- * @return a long integer representing the number of edges
- */
+ /** @return a long integer representing the number of edges */
def numberOfEdges(): Long = {
jgraph.numberOfEdges()
}
- /**
- * @return The IDs of the vertices as DataSet
- */
+ /** @return The IDs of the vertices as DataSet */
def getVertexIds(): DataSet[K] = {
wrap(jgraph.getVertexIds)
}
- /**
- * @return The IDs of the edges as DataSet
- */
+ /** @return The IDs of the edges as DataSet */
def getEdgeIds(): DataSet[(K, K)] = {
wrap(jgraph.getEdgeIds).map((i: jtuple.Tuple2[K, K]) => (i.f0, i.f1))
}
/**
- * Adds the input vertex to the graph. If the vertex already
- * exists in the graph, it will not be added again.
+ * Adds the input vertex to the graph. If the vertex already exists in the graph, it will not be
+ * added again.
*
- * @param vertex the vertex to be added
- * @return the new graph containing the existing vertices as well as the one just added
+ * @param vertex
+ * the vertex to be added
+ * @return
+ * the new graph containing the existing vertices as well as the one just added
*/
def addVertex(vertex: Vertex[K, VV]) = {
wrapGraph(jgraph.addVertex(vertex))
}
/**
- * Adds the list of vertices, passed as input, to the graph.
- * If the vertices already exist in the graph, they will not be added once more.
+ * Adds the list of vertices, passed as input, to the graph. If the vertices already exist in the
+ * graph, they will not be added once more.
*
- * @param vertices the list of vertices to add
- * @return the new graph containing the existing and newly added vertices
+ * @param vertices
+ * the list of vertices to add
+ * @return
+ * the new graph containing the existing and newly added vertices
*/
def addVertices(vertices: List[Vertex[K, VV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.addVertices(vertices.asJava))
@@ -956,25 +1051,30 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Adds the given list edges to the graph.
*
- * When adding an edge for a non-existing set of vertices,
- * the edge is considered invalid and ignored.
+ * When adding an edge for a non-existing set of vertices, the edge is considered invalid and
+ * ignored.
*
- * @param edges the data set of edges to be added
- * @return a new graph containing the existing edges plus the newly added edges.
+ * @param edges
+ * the data set of edges to be added
+ * @return
+ * a new graph containing the existing edges plus the newly added edges.
*/
def addEdges(edges: List[Edge[K, EV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.addEdges(edges.asJava))
}
/**
- * Adds the given edge to the graph. If the source and target vertices do
- * not exist in the graph, they will also be added.
+ * Adds the given edge to the graph. If the source and target vertices do not exist in the graph,
+ * they will also be added.
*
- * @param source the source vertex of the edge
- * @param target the target vertex of the edge
- * @param edgeValue the edge value
- * @return the new graph containing the existing vertices and edges plus the
- * newly added edge
+ * @param source
+ * the source vertex of the edge
+ * @param target
+ * the target vertex of the edge
+ * @param edgeValue
+ * the edge value
+ * @return
+ * the new graph containing the existing vertices and edges plus the newly added edge
*/
def addEdge(source: Vertex[K, VV], target: Vertex[K, VV], edgeValue: EV) = {
wrapGraph(jgraph.addEdge(source, target, edgeValue))
@@ -983,9 +1083,11 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes the given vertex and its edges from the graph.
*
- * @param vertex the vertex to remove
- * @return the new graph containing the existing vertices and edges without
- * the removed vertex and its edges
+ * @param vertex
+ * the vertex to remove
+ * @return
+ * the new graph containing the existing vertices and edges without the removed vertex and its
+ * edges
*/
def removeVertex(vertex: Vertex[K, VV]) = {
wrapGraph(jgraph.removeVertex(vertex))
@@ -994,9 +1096,11 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes the given vertex and its edges from the graph.
*
- * @param vertices list of vertices to remove
- * @return the new graph containing the existing vertices and edges without
- * the removed vertex and its edges
+ * @param vertices
+ * list of vertices to remove
+ * @return
+ * the new graph containing the existing vertices and edges without the removed vertex and its
+ * edges
*/
def removeVertices(vertices: List[Vertex[K, VV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.removeVertices(vertices.asJava))
@@ -1005,9 +1109,10 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes all edges that match the given edge from the graph.
*
- * @param edge the edge to remove
- * @return the new graph containing the existing vertices and edges without
- * the removed edges
+ * @param edge
+ * the edge to remove
+ * @return
+ * the new graph containing the existing vertices and edges without the removed edges
*/
def removeEdge(edge: Edge[K, EV]) = {
wrapGraph(jgraph.removeEdge(edge))
@@ -1016,31 +1121,36 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes all the edges that match the edges in the given data set from the graph.
*
- * @param edges the list of edges to be removed
- * @return a new graph where the edges have been removed and in which the vertices remained intact
+ * @param edges
+ * the list of edges to be removed
+ * @return
+ * a new graph where the edges have been removed and in which the vertices remained intact
*/
def removeEdges(edges: List[Edge[K, EV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.removeEdges(edges.asJava))
}
/**
- * Performs union on the vertices and edges sets of the input graphs
- * removing duplicate vertices but maintaining duplicate edges.
+ * Performs union on the vertices and edges sets of the input graphs removing duplicate vertices
+ * but maintaining duplicate edges.
*
- * @param graph the graph to perform union with
- * @return a new graph
+ * @param graph
+ * the graph to perform union with
+ * @return
+ * a new graph
*/
def union(graph: Graph[K, VV, EV]) = {
wrapGraph(jgraph.union(graph.getWrappedGraph))
}
/**
- * Performs Difference on the vertex and edge sets of the input graphs
- * removes common vertices and edges. If a source/target vertex is removed,
- * its corresponding edge will also be removed
+ * Performs Difference on the vertex and edge sets of the input graphs removes common vertices and
+ * edges. If a source/target vertex is removed, its corresponding edge will also be removed
*
- * @param graph the graph to perform difference with
- * @return a new graph where the common vertices and edges have been removed
+ * @param graph
+ * the graph to perform difference with
+ * @return
+ * a new graph where the common vertices and edges have been removed
*/
def difference(graph: Graph[K, VV, EV]) = {
wrapGraph(jgraph.difference(graph.getWrappedGraph))
@@ -1048,196 +1158,240 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Performs intersect on the edge sets of the input graphs. Edges are considered equal, if they
- * have the same source identifier, target identifier and edge value.
- * <p>
- * The method computes pairs of equal edges from the input graphs. If the same edge occurs
- * multiple times in the input graphs, there will be multiple edge pairs to be considered. Each
- * edge instance can only be part of one pair. If the given parameter `distinctEdges` is set
- * to `true`, there will be exactly one edge in the output graph representing all pairs of
- * equal edges. If the parameter is set to `false`, both edges of each pair will be in the
- * output.
- * <p>
- * Vertices in the output graph will have no vertex values.
+ * have the same source identifier, target identifier and edge value. <p> The method computes
+ * pairs of equal edges from the input graphs. If the same edge occurs multiple times in the input
+ * graphs, there will be multiple edge pairs to be considered. Each edge instance can only be part
+ * of one pair. If the given parameter `distinctEdges` is set to `true`, there will be exactly one
+ * edge in the output graph representing all pairs of equal edges. If the parameter is set to
+ * `false`, both edges of each pair will be in the output. <p> Vertices in the output graph will
+ * have no vertex values.
*
- * @param graph the graph to perform intersect with
- * @param distinctEdges if set to { @code true}, there will be exactly one edge in the output
- * graph representing all pairs of equal edges, otherwise, for each pair,
- * both edges will be in the output graph
- * @return a new graph which contains only common vertices and edges from the input graphs
+ * @param graph
+ * the graph to perform intersect with
+ * @param distinctEdges
+ * if set to { @code true}, there will be exactly one edge in the output graph representing all
+ * pairs of equal edges, otherwise, for each pair, both edges will be in the output graph
+ * @return
+ * a new graph which contains only common vertices and edges from the input graphs
*/
def intersect(graph: Graph[K, VV, EV], distinctEdges: Boolean): Graph[K, NullValue, EV] = {
wrapGraph(jgraph.intersect(graph.getWrappedGraph, distinctEdges))
}
/**
- * Compute a reduce transformation over the neighbors' vertex values of each vertex.
- * For each vertex, the transformation consecutively calls a
- * [[ReduceNeighborsFunction]] until only a single value for each vertex remains.
- * The [[ReduceNeighborsFunction]] combines a pair of neighbor vertex values
- * into one new value of the same type.
- *
- * @param reduceNeighborsFunction the reduce function to apply to the neighbors of each vertex.
- * @param direction the edge direction (in-, out-, all-)
- * @return a Dataset of Tuple2, with one tuple per vertex.
- * The first field of the Tuple2 is the vertex ID and the second field
- * is the aggregate value computed by the provided [[ReduceNeighborsFunction]].
- */
- def reduceOnNeighbors(reduceNeighborsFunction: ReduceNeighborsFunction[VV], direction:
- EdgeDirection): DataSet[(K, VV)] = {
+ * Compute a reduce transformation over the neighbors' vertex values of each vertex. For each
+ * vertex, the transformation consecutively calls a [[ReduceNeighborsFunction]] until only a
+ * single value for each vertex remains. The [[ReduceNeighborsFunction]] combines a pair of
+ * neighbor vertex values into one new value of the same type.
+ *
+ * @param reduceNeighborsFunction
+ * the reduce function to apply to the neighbors of each vertex.
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @return
+ * a Dataset of Tuple2, with one tuple per vertex. The first field of the Tuple2 is the vertex
+ * ID and the second field is the aggregate value computed by the provided
+ * [[ReduceNeighborsFunction]].
+ */
+ def reduceOnNeighbors(
+ reduceNeighborsFunction: ReduceNeighborsFunction[VV],
+ direction: EdgeDirection): DataSet[(K, VV)] = {
wrap(jgraph.reduceOnNeighbors(reduceNeighborsFunction, direction)).map(
(i: jtuple.Tuple2[K, VV]) => (i.f0, i.f1))
}
/**
- * Compute a reduce transformation over the neighbors' vertex values of each vertex.
- * For each vertex, the transformation consecutively calls a
- * [[ReduceNeighborsFunction]] until only a single value for each vertex remains.
- * The [[ReduceNeighborsFunction]] combines a pair of neighbor vertex values
- * into one new value of the same type.
- *
- * @param reduceEdgesFunction the reduce function to apply to the edges of each vertex.
- * @param direction the edge direction (in-, out-, all-)
- * @return a Dataset of Tuple2, with one tuple per vertex.
- * The first field of the Tuple2 is the vertex ID and the second field
- * is the aggregate value computed by the provided [[ReduceNeighborsFunction]].
- */
- def reduceOnEdges(reduceEdgesFunction: ReduceEdgesFunction[EV], direction: EdgeDirection):
- DataSet[(K, EV)] = {
+ * Compute a reduce transformation over the neighbors' vertex values of each vertex. For each
+ * vertex, the transformation consecutively calls a [[ReduceNeighborsFunction]] until only a
+ * single value for each vertex remains. The [[ReduceNeighborsFunction]] combines a pair of
+ * neighbor vertex values into one new value of the same type.
+ *
+ * @param reduceEdgesFunction
+ * the reduce function to apply to the edges of each vertex.
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @return
+ * a Dataset of Tuple2, with one tuple per vertex. The first field of the Tuple2 is the vertex
+ * ID and the second field is the aggregate value computed by the provided
+ * [[ReduceNeighborsFunction]].
+ */
+ def reduceOnEdges(
+ reduceEdgesFunction: ReduceEdgesFunction[EV],
+ direction: EdgeDirection): DataSet[(K, EV)] = {
wrap(jgraph.reduceOnEdges(reduceEdgesFunction, direction)).map(
(i: jtuple.Tuple2[K, EV]) => (i.f0, i.f1))
}
/**
- * @param algorithm the algorithm to run on the Graph
- * @return the result of the graph algorithm
+ * @param algorithm
+ * the algorithm to run on the Graph
+ * @return
+ * the result of the graph algorithm
*/
- def run[T: TypeInformation : ClassTag](algorithm: GraphAlgorithm[K, VV, EV, T]):
- T = {
+ def run[T: TypeInformation: ClassTag](algorithm: GraphAlgorithm[K, VV, EV, T]): T = {
jgraph.run(algorithm)
}
/**
- * A GraphAnalytic is similar to a GraphAlgorithm but is terminal and results
- * are retrieved via accumulators. A Flink program has a single point of
- * execution. A GraphAnalytic defers execution to the user to allow composing
- * multiple analytics and algorithms into a single program.
+ * A GraphAnalytic is similar to a GraphAlgorithm but is terminal and results are retrieved via
+ * accumulators. A Flink program has a single point of execution. A GraphAnalytic defers execution
+ * to the user to allow composing multiple analytics and algorithms into a single program.
*
- * @param analytic the analytic to run on the Graph
+ * @param analytic
+ * the analytic to run on the Graph
*/
- def run[T: TypeInformation : ClassTag](analytic: GraphAnalytic[K, VV, EV, T]):
- GraphAnalytic[K, VV, EV, T] = {
+ def run[T: TypeInformation: ClassTag](
+ analytic: GraphAnalytic[K, VV, EV, T]): GraphAnalytic[K, VV, EV, T] = {
jgraph.run(analytic)
analytic
}
/**
- * Runs a scatter-gather iteration on the graph.
- * No configuration options are provided.
+ * Runs a scatter-gather iteration on the graph. No configuration options are provided.
*
- * @param scatterFunction the scatter function
- * @param gatherFunction the gather function
- * @param maxIterations maximum number of iterations to perform
- * @return the updated Graph after the scatter-gather iteration has converged or
- * after maximumNumberOfIterations.
+ * @param scatterFunction
+ * the scatter function
+ * @param gatherFunction
+ * the gather function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @return
+ * the updated Graph after the scatter-gather iteration has converged or after
+ * maximumNumberOfIterations.
*/
- def runScatterGatherIteration[M](scatterFunction: ScatterFunction[K, VV, M, EV],
- gatherFunction: SpargelGatherFunction[K, VV, M],
- maxIterations: Int): Graph[K, VV, EV] = {
- wrapGraph(jgraph.runScatterGatherIteration(scatterFunction, gatherFunction,
- maxIterations))
+ def runScatterGatherIteration[M](
+ scatterFunction: ScatterFunction[K, VV, M, EV],
+ gatherFunction: SpargelGatherFunction[K, VV, M],
+ maxIterations: Int): Graph[K, VV, EV] = {
+ wrapGraph(jgraph.runScatterGatherIteration(scatterFunction, gatherFunction, maxIterations))
}
/**
* Runs a scatter-gather iteration on the graph with configuration options.
*
- * @param scatterFunction the scatter function
- * @param gatherFunction the gather function
- * @param maxIterations maximum number of iterations to perform
- * @param parameters the iteration configuration parameters
- * @return the updated Graph after the scatter-gather iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runScatterGatherIteration[M](scatterFunction: ScatterFunction[K, VV, M, EV],
- gatherFunction: SpargelGatherFunction[K, VV, M],
- maxIterations: Int, parameters: ScatterGatherConfiguration):
- Graph[K, VV, EV] = {
- wrapGraph(jgraph.runScatterGatherIteration(scatterFunction, gatherFunction,
- maxIterations, parameters))
- }
-
- /**
- * Runs a Gather-Sum-Apply iteration on the graph.
- * No configuration options are provided.
+ * @param scatterFunction
+ * the scatter function
+ * @param gatherFunction
+ * the gather function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @param parameters
+ * the iteration configuration parameters
+ * @return
+ * the updated Graph after the scatter-gather iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runScatterGatherIteration[M](
+ scatterFunction: ScatterFunction[K, VV, M, EV],
+ gatherFunction: SpargelGatherFunction[K, VV, M],
+ maxIterations: Int,
+ parameters: ScatterGatherConfiguration): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runScatterGatherIteration(scatterFunction, gatherFunction, maxIterations, parameters))
+ }
+
+ /**
+ * Runs a Gather-Sum-Apply iteration on the graph. No configuration options are provided.
*
- * @param gatherFunction the gather function collects information about adjacent
- * vertices and edges
- * @param sumFunction the sum function aggregates the gathered information
- * @param applyFunction the apply function updates the vertex values with the aggregates
- * @param maxIterations maximum number of iterations to perform
- * @tparam M the intermediate type used between gather, sum and apply
- * @return the updated Graph after the gather-sum-apply iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runGatherSumApplyIteration[M](gatherFunction: GSAGatherFunction[VV, EV, M], sumFunction:
- SumFunction[VV, EV, M], applyFunction: ApplyFunction[K, VV, M], maxIterations: Int): Graph[K,
- VV, EV] = {
- wrapGraph(jgraph.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction,
- maxIterations))
+ * @param gatherFunction
+ * the gather function collects information about adjacent vertices and edges
+ * @param sumFunction
+ * the sum function aggregates the gathered information
+ * @param applyFunction
+ * the apply function updates the vertex values with the aggregates
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @tparam M
+ * the intermediate type used between gather, sum and apply
+ * @return
+ * the updated Graph after the gather-sum-apply iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runGatherSumApplyIteration[M](
+ gatherFunction: GSAGatherFunction[VV, EV, M],
+ sumFunction: SumFunction[VV, EV, M],
+ applyFunction: ApplyFunction[K, VV, M],
+ maxIterations: Int): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction, maxIterations))
}
/**
* Runs a Gather-Sum-Apply iteration on the graph with configuration options.
*
- * @param gatherFunction the gather function collects information about adjacent
- * vertices and edges
- * @param sumFunction the sum function aggregates the gathered information
- * @param applyFunction the apply function updates the vertex values with the aggregates
- * @param maxIterations maximum number of iterations to perform
- * @param parameters the iteration configuration parameters
- * @tparam M the intermediate type used between gather, sum and apply
- * @return the updated Graph after the gather-sum-apply iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runGatherSumApplyIteration[M](gatherFunction: GSAGatherFunction[VV, EV, M], sumFunction:
- SumFunction[VV, EV, M], applyFunction: ApplyFunction[K, VV, M], maxIterations: Int,
- parameters: GSAConfiguration): Graph[K, VV, EV] = {
- wrapGraph(jgraph.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction,
- maxIterations, parameters))
- }
-
- /**
- * Runs a vertex-centric iteration on the graph.
- * No configuration options are provided.
+ * @param gatherFunction
+ * the gather function collects information about adjacent vertices and edges
+ * @param sumFunction
+ * the sum function aggregates the gathered information
+ * @param applyFunction
+ * the apply function updates the vertex values with the aggregates
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @param parameters
+ * the iteration configuration parameters
+ * @tparam M
+ * the intermediate type used between gather, sum and apply
+ * @return
+ * the updated Graph after the gather-sum-apply iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runGatherSumApplyIteration[M](
+ gatherFunction: GSAGatherFunction[VV, EV, M],
+ sumFunction: SumFunction[VV, EV, M],
+ applyFunction: ApplyFunction[K, VV, M],
+ maxIterations: Int,
+ parameters: GSAConfiguration): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runGatherSumApplyIteration(
+ gatherFunction,
+ sumFunction,
+ applyFunction,
+ maxIterations,
+ parameters))
+ }
+
+ /**
+ * Runs a vertex-centric iteration on the graph. No configuration options are provided.
*
- * @param computeFunction the compute function
- * @param combineFunction the optional message combiner function
- * @param maxIterations maximum number of iterations to perform
- * @return the updated Graph after the vertex-centric iteration has converged or
- * after maximumNumberOfIterations.
+ * @param computeFunction
+ * the compute function
+ * @param combineFunction
+ * the optional message combiner function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @return
+ * the updated Graph after the vertex-centric iteration has converged or after
+ * maximumNumberOfIterations.
*/
- def runVertexCentricIteration[M](computeFunction: ComputeFunction[K, VV, EV, M],
- combineFunction: MessageCombiner[K, M],
- maxIterations: Int): Graph[K, VV, EV] = {
- wrapGraph(jgraph.runVertexCentricIteration(computeFunction, combineFunction,
- maxIterations))
+ def runVertexCentricIteration[M](
+ computeFunction: ComputeFunction[K, VV, EV, M],
+ combineFunction: MessageCombiner[K, M],
+ maxIterations: Int): Graph[K, VV, EV] = {
+ wrapGraph(jgraph.runVertexCentricIteration(computeFunction, combineFunction, maxIterations))
}
/**
* Runs a vertex-centric iteration on the graph with configuration options.
*
- * @param computeFunction the compute function
- * @param combineFunction the optional message combiner function
- * @param maxIterations maximum number of iterations to perform
- * @param parameters the iteration configuration parameters
- * @return the updated Graph after the vertex-centric iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runVertexCentricIteration[M](computeFunction: ComputeFunction[K, VV, EV, M],
- combineFunction: MessageCombiner[K, M],
- maxIterations: Int, parameters: VertexCentricConfiguration):
- Graph[K, VV, EV] = {
- wrapGraph(jgraph.runVertexCentricIteration(computeFunction, combineFunction,
- maxIterations, parameters))
+ * @param computeFunction
+ * the compute function
+ * @param combineFunction
+ * the optional message combiner function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @param parameters
+ * the iteration configuration parameters
+ * @return
+ * the updated Graph after the vertex-centric iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runVertexCentricIteration[M](
+ computeFunction: ComputeFunction[K, VV, EV, M],
+ combineFunction: MessageCombiner[K, M],
+ maxIterations: Int,
+ parameters: VertexCentricConfiguration): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runVertexCentricIteration(computeFunction, combineFunction, maxIterations, parameters))
}
def validate(validator: GraphValidator[K, VV, EV]): Boolean = {
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala
index ca15dabdcca..9ccd18e27b8 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala
@@ -15,22 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
import org.apache.flink.api.java.tuple.Tuple3
import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.util.Collector
-
-abstract class NeighborsFunction[K, VV, EV, T] extends org.apache.flink.graph
-.NeighborsFunction[K, VV, EV, T] {
+abstract class NeighborsFunction[K, VV, EV, T]
+ extends org.apache.flink.graph.NeighborsFunction[K, VV, EV, T] {
def iterateNeighbors(neighbors: Iterable[(K, Edge[K, EV], Vertex[K, VV])], out: Collector[T])
- override def iterateNeighbors(neighbors: java.lang.Iterable[Tuple3[K, Edge[K, EV], Vertex[K,
- VV]]], out: Collector[T]) = {
- val scalaIterable = scala.collection.JavaConversions.iterableAsScalaIterable(neighbors)
+ override def iterateNeighbors(
+ neighbors: java.lang.Iterable[Tuple3[K, Edge[K, EV], Vertex[K, VV]]],
+ out: Collector[T]) = {
+ val scalaIterable = scala.collection.JavaConversions
+ .iterableAsScalaIterable(neighbors)
.map(jtuple => (jtuple.f0, jtuple.f1, jtuple.f2))
iterateNeighbors(scalaIterable, out)
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala
index cefc277c00a..d9fdd13af10 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala
@@ -15,25 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
-import java.lang
-
import org.apache.flink.api.java.tuple.Tuple2
import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.util.Collector
+import java.lang
-abstract class NeighborsFunctionWithVertexValue[K, VV, EV, T] extends org.apache.flink.graph
-.NeighborsFunctionWithVertexValue[K, VV, EV, T] {
+abstract class NeighborsFunctionWithVertexValue[K, VV, EV, T]
+ extends org.apache.flink.graph.NeighborsFunctionWithVertexValue[K, VV, EV, T] {
- def iterateNeighbors(vertex: Vertex[K, VV], neighbors: Iterable[(Edge[K, EV], Vertex[K, VV])
- ], out: Collector[T]): Unit
+ def iterateNeighbors(
+ vertex: Vertex[K, VV],
+ neighbors: Iterable[(Edge[K, EV], Vertex[K, VV])],
+ out: Collector[T]): Unit
- override def iterateNeighbors(vertex: Vertex[K, VV], neighbors: lang.Iterable[Tuple2[Edge[K,
- EV], Vertex[K, VV]]], out: Collector[T]): Unit = {
- val scalaIterable = scala.collection.JavaConversions.iterableAsScalaIterable(neighbors)
+ override def iterateNeighbors(
+ vertex: Vertex[K, VV],
+ neighbors: lang.Iterable[Tuple2[Edge[K, EV], Vertex[K, VV]]],
+ out: Collector[T]): Unit = {
+ val scalaIterable = scala.collection.JavaConversions
+ .iterableAsScalaIterable(neighbors)
.map(jtuple => (jtuple.f0, jtuple.f1))
iterateNeighbors(vertex, scalaIterable, out)
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala
index 83b6b6d22dc..24a5726d31a 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph
import org.apache.flink.api.common.typeinfo.TypeInformation
@@ -23,14 +22,11 @@ import org.apache.flink.graph.{Graph => JGraph}
import _root_.scala.reflect.ClassTag
-
package object scala {
private[flink] def wrapGraph[
- K: TypeInformation : ClassTag,
- VV: TypeInformation : ClassTag,
- EV: TypeInformation : ClassTag](
- javagraph: JGraph[K, VV, EV])
- : scala.Graph[K, VV, EV] = {
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](javagraph: JGraph[K, VV, EV]): scala.Graph[K, VV, EV] = {
new scala.Graph[K, VV, EV](javagraph)
}
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala
index 909dbb49aaf..9269087b4e3 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.utils
import org.apache.flink.api.common.functions.MapFunction
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala
index fd6b8c535bd..4e4627e35c9 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala
... 203010 lines suppressed ...
[flink] 04/05: [FLINK-27317][build] Don't fork build for attaching sources
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
commit bea6bafac98770da20ef8092838a9d79768b2cf5
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Apr 20 09:57:31 2022 +0200
[FLINK-27317][build] Don't fork build for attaching sources
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 849829b0b6a..b85a2bf01a8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1238,7 +1238,7 @@ under the License.
<execution>
<id>attach-sources</id>
<goals>
- <goal>jar</goal>
+ <goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
[flink] 01/05: [FLINK-26553][build] Add scalafmt for formatting the Scala codebase
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 033273a658a251d3d8734abee4739817839e139f
Author: slinkydeveloper <fr...@gmail.com>
AuthorDate: Wed Mar 9 15:48:16 2022 +0100
[FLINK-26553][build] Add scalafmt for formatting the Scala codebase
Run 'mvn spotless:apply' for auto-formatting the code base. Usually,
IntelliJ should pick it up automatically if the Scala plugin is installed.
This closes #19025.
---
.editorconfig | 4 -
.scalafmt.conf | 70 ++++++++++
.../flink-hadoop-compatibility/pom.xml | 9 --
flink-connectors/flink-hcatalog/pom.xml | 10 --
.../flink-end-to-end-tests-scala/pom.xml | 11 +-
.../test-scripts/test_kubernetes_itcases.sh | 4 +-
.../test-scripts/test_streaming_gcp_pubsub.sh | 5 +-
flink-examples/flink-examples-batch/pom.xml | 9 --
flink-examples/flink-examples-streaming/pom.xml | 9 --
flink-libraries/flink-cep-scala/pom.xml | 9 --
flink-libraries/flink-gelly-examples/pom.xml | 9 --
flink-libraries/flink-gelly-scala/pom.xml | 9 --
.../scala/org/apache/flink/graph/scala/Graph.scala | 2 -
flink-scala/pom.xml | 9 --
.../apache/flink/api/scala/ClosureCleaner.scala | 4 -
flink-streaming-scala/pom.xml | 9 --
flink-table/flink-table-api-scala-bridge/pom.xml | 9 --
flink-table/flink-table-api-scala/pom.xml | 9 --
flink-table/flink-table-planner/pom.xml | 9 --
flink-tests/pom.xml | 9 --
flink-yarn-tests/pom.xml | 9 --
pom.xml | 64 ++++-----
tools/ci/compile.sh | 6 +-
tools/maven/scalastyle-config.xml | 146 ---------------------
24 files changed, 105 insertions(+), 338 deletions(-)
diff --git a/.editorconfig b/.editorconfig
index 2e260749d26..333c2a16580 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -267,10 +267,6 @@ ij_java_wrap_first_method_in_call_chain = true
indent_style = tab
indent_size = 4
-[*.scala]
-indent_style = space
-indent_size = 2
-
[*.py]
indent_style = space
indent_size = 4
diff --git a/.scalafmt.conf b/.scalafmt.conf
new file mode 100644
index 00000000000..a2ea10cc981
--- /dev/null
+++ b/.scalafmt.conf
@@ -0,0 +1,70 @@
+runner.dialect = scala212
+
+# Version is required to make sure IntelliJ picks the right version
+version = 3.4.3
+preset = default
+
+# Max column
+maxColumn = 100
+
+# This parameter simply says the .stripMargin method was not redefined by the user to assign
+# special meaning to indentation preceding the | character. Hence, that indentation can be modified.
+assumeStandardLibraryStripMargin = true
+align.stripMargin = true
+
+# Align settings
+align.preset = none
+align.closeParenSite = false
+align.openParenCallSite = false
+danglingParentheses.defnSite = false
+danglingParentheses.callSite = false
+danglingParentheses.ctrlSite = true
+danglingParentheses.tupleSite = false
+align.openParenCallSite = false
+align.openParenDefnSite = false
+align.openParenTupleSite = false
+
+# Newlines
+newlines.alwaysBeforeElseAfterCurlyIf = false
+newlines.beforeCurlyLambdaParams = multiline # Newline before lambda params
+newlines.afterCurlyLambdaParams = squash # No newline after lambda params
+newlines.inInterpolation = "avoid"
+newlines.avoidInResultType = true
+optIn.annotationNewlines = true
+
+# Scaladoc
+docstrings.style = Asterisk # Javadoc style
+docstrings.removeEmpty = true
+docstrings.oneline = fold
+docstrings.forceBlankLineBefore = true
+
+# Indentation
+indent.extendSite = 2 # This makes sure extend is not indented as the ctor parameters
+
+# Rewrites
+rewrite.rules = [AvoidInfix, Imports, RedundantBraces, SortModifiers]
+
+# Imports
+rewrite.imports.sort = scalastyle
+rewrite.imports.groups = [
+ ["org.apache.flink\\..*"],
+ ["org.apache.flink.shaded\\..*"],
+ [".*"],
+ ["javax\\..*"],
+ ["java\\..*"],
+ ["scala\\..*"]
+]
+rewrite.imports.contiguousGroups = no
+importSelectors = singleline # Imports in a single line, like IntelliJ
+
+# Remove redundant braces in string interpolation.
+rewrite.redundantBraces.stringInterpolation = true
+rewrite.redundantBraces.defnBodies = false
+rewrite.redundantBraces.generalExpressions = false
+rewrite.redundantBraces.ifElseExpressions = false
+rewrite.redundantBraces.methodBodies = false
+rewrite.redundantBraces.includeUnitMethods = false
+rewrite.redundantBraces.maxBreaks = 1
+
+# Remove trailing commas
+rewrite.trailingCommas.style = "never"
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 54c1b966faf..7ec965570a5 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -178,15 +178,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<!-- Add test classes to test jar in order to test WritableTypeInfo. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
index 4351c85ed4a..fa222d8bb2b 100644
--- a/flink-connectors/flink-hcatalog/pom.xml
+++ b/flink-connectors/flink-hcatalog/pom.xml
@@ -192,16 +192,6 @@ under the License.
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
</plugins>
</build>
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
index 0a514c8165e..03a94cf5371 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
@@ -202,15 +202,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<!-- Add Scala test classes to test jar in order to test Scala type information. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -226,4 +217,4 @@ under the License.
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh
index 6ff1a9fff73..18dca21fe3e 100755
--- a/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh
+++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh
@@ -32,7 +32,7 @@ MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFi
function run_mvn_test {
local test_class=$1
- run_mvn test $MVN_LOGGING_OPTIONS -Dtest=${test_class}
+ run_mvn test -pl flink-kubernetes $MVN_LOGGING_OPTIONS -Dtest=${test_class}
EXIT_CODE=$?
if [ $EXIT_CODE != 0 ]; then
@@ -41,7 +41,7 @@ function run_mvn_test {
fi
}
-cd $END_TO_END_DIR/../flink-kubernetes
+cd $END_TO_END_DIR/..
# Run the ITCases
run_mvn_test org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClientITCase
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_gcp_pubsub.sh b/flink-end-to-end-tests/test-scripts/test_streaming_gcp_pubsub.sh
index 625d818c442..99f11003126 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_gcp_pubsub.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_gcp_pubsub.sh
@@ -18,7 +18,4 @@
################################################################################
# This test is a Java end to end test, but it requires Docker. Therefore, we run it from bash.
-
-cd "${END_TO_END_DIR}/flink-connector-gcp-pubsub-emulator-tests"
-
-run_mvn test -DskipTests=false
+run_mvn test -DskipTests=false -pl flink-end-to-end-tests/flink-connector-gcp-pubsub-emulator-tests
diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml
index 6ead88d94d2..13eecd44db1 100644
--- a/flink-examples/flink-examples-batch/pom.xml
+++ b/flink-examples/flink-examples-batch/pom.xml
@@ -164,15 +164,6 @@ under the License.
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
<!-- create the example JAR files -->
<plugin>
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index 744463d422f..5d1d0dd5ef2 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -143,15 +143,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<!-- self-contained jars for each example -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
index ac0dae0f6d9..19d9c634179 100644
--- a/flink-libraries/flink-cep-scala/pom.xml
+++ b/flink-libraries/flink-cep-scala/pom.xml
@@ -122,15 +122,6 @@ under the License.
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
<!-- Adding scala source directories to build path for Eclipse and generating the sources jar -->
<plugin>
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index d53855f7093..a5dc0070d3d 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -236,15 +236,6 @@
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
</plugins>
</build>
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index b99752fd686..be918e10c2f 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -199,15 +199,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<!-- build a jar-with-dependencies, to be included in the 'opt' build folder -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
index 4609c5e0095..df7b8c10822 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
@@ -218,7 +218,6 @@ object Graph {
* @tparam EV Edge value type
* @return Graph with vertices and edges read from the given files.
*/
- // scalastyle:off
// This method exceeds the max allowed number of parameters -->
def fromCsvReader[
K: TypeInformation : ClassTag,
@@ -290,7 +289,6 @@ object Graph {
}
}
}
- // scalastyle:on
}
/**
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 3f8b5a1e63a..c4de68fb589 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -252,15 +252,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<!-- Add Scala test classes to test jar in order to test Scala type information. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala
index 6e6e536fd14..3ad71ea5d4b 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala
@@ -381,10 +381,8 @@ object ClosureCleaner {
} else {
LOG.debug(s"Cleaning lambda: ${lambdaFunc.get.getImplMethodName}")
- // scalastyle:off classforname
val captClass = Class.forName(lambdaFunc.get.getCapturingClass.replace('/', '.'),
false, Thread.currentThread.getContextClassLoader)
- // scalastyle:on classforname
// Fail fast if we detect return statements in closures
getClassReader(captClass)
.accept(new ReturnStatementFinder(Some(lambdaFunc.get.getImplMethodName)), 0)
@@ -593,12 +591,10 @@ private class InnerClosureFinder(output: Set[Class[_]]) extends ClassVisitor(ASM
if (op == INVOKESPECIAL && name == "<init>" && argTypes.length > 0
&& argTypes(0).toString.startsWith("L") // is it an object?
&& argTypes(0).getInternalName == myName) {
- // scalastyle:off classforname
output += Class.forName(
owner.replace('/', '.'),
false,
Thread.currentThread.getContextClassLoader)
- // scalastyle:on classforname
}
}
}
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index bbee28d315c..d6533c3482a 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -238,15 +238,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<!-- activate API compatibility checks -->
<plugin>
<groupId>com.github.siom79.japicmp</groupId>
diff --git a/flink-table/flink-table-api-scala-bridge/pom.xml b/flink-table/flink-table-api-scala-bridge/pom.xml
index 9dd3d220721..38381c1ae5a 100644
--- a/flink-table/flink-table-api-scala-bridge/pom.xml
+++ b/flink-table/flink-table-api-scala-bridge/pom.xml
@@ -103,15 +103,6 @@ under the License.
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
diff --git a/flink-table/flink-table-api-scala/pom.xml b/flink-table/flink-table-api-scala/pom.xml
index 84b79e716ce..fde30cccf6b 100644
--- a/flink-table/flink-table-api-scala/pom.xml
+++ b/flink-table/flink-table-api-scala/pom.xml
@@ -114,15 +114,6 @@ under the License.
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index 469a7a3d591..e059cbdbf00 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -398,15 +398,6 @@ under the License.
</configuration>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 1acd328ea7e..0cfbe29ad93 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -374,15 +374,6 @@ under the License.
</execution>
</executions>
</plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index 0bb1ac219b6..1b1e1941287 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -344,15 +344,6 @@ under the License.
</executions>
</plugin>
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
diff --git a/pom.xml b/pom.xml
index edd9a3900d3..a03c6ba60e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -157,7 +157,7 @@ under the License.
<hivemetastore.hadoop.version>2.7.5</hivemetastore.hadoop.version>
<japicmp.referenceVersion>1.15.0</japicmp.referenceVersion>
<japicmp.outputDir>tools/japicmp-output</japicmp.outputDir>
- <spotless.version>2.4.2</spotless.version>
+ <spotless.version>2.13.0</spotless.version>
<!-- Can be set to any value to reproduce a specific build. -->
<test.randomization.seed/>
@@ -1068,13 +1068,6 @@ under the License.
<skip>true</skip>
</configuration>
</plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
@@ -1526,7 +1519,6 @@ under the License.
<!-- Generated content -->
<exclude>out/**</exclude>
<exclude>**/target/**</exclude>
- <exclude>**/scalastyle-output.xml</exclude>
<exclude>build-target/**</exclude>
<exclude>docs/layouts/shortcodes/generated/**</exclude>
<exclude>docs/static/generated/**</exclude>
@@ -1967,6 +1959,34 @@ under the License.
<removeUnusedImports />
</java>
+ <scala>
+ <scalafmt>
+ <version>3.4.3</version>
+ <!-- This file is in the root of the project to make sure IntelliJ picks it up automatically -->
+ <file>.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+</content>
+ <delimiter>package </delimiter>
+ </licenseHeader>
+ </scala>
</configuration>
<executions>
<execution>
@@ -2062,32 +2082,6 @@ under the License.
</executions>
</plugin>
- <!-- configure scala style -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <version>1.0.0</version>
- <executions>
- <execution>
- <phase>validate</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
- <outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>
- <inputEncoding>UTF-8</inputEncoding>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- </plugin>
-
<!-- set scala maven plugin version -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh
index e6013f9650f..591e749d514 100755
--- a/tools/ci/compile.sh
+++ b/tools/ci/compile.sh
@@ -71,7 +71,7 @@ echo "============ Checking Javadocs ============"
# use the same invocation as on buildbot (https://svn.apache.org/repos/infra/infrastructure/buildbot/aegis/buildmaster/master1/projects/flink.conf)
run_mvn javadoc:aggregate -Paggregate-scaladoc -DadditionalJOption='-Xdoclint:none' \
- -Dmaven.javadoc.failOnError=false -Dcheckstyle.skip=true -Denforcer.skip=true \
+ -Dmaven.javadoc.failOnError=false -Dcheckstyle.skip=true -Denforcer.skip=true -Dspotless.skip=true \
-Dheader=someTestHeader > javadoc.out
EXIT_CODE=$?
if [ $EXIT_CODE != 0 ] ; then
@@ -82,15 +82,13 @@ fi
echo "============ Checking Scaladocs ============"
-cd flink-scala
-run_mvn scala:doc 2> scaladoc.out
+run_mvn scala:doc -Dcheckstyle.skip=true -Denforcer.skip=true -Dspotless.skip=true -pl flink-scala 2> scaladoc.out
EXIT_CODE=$?
if [ $EXIT_CODE != 0 ] ; then
echo "ERROR in Scaladocs. Printing full output:"
cat scaladoc.out ; rm scaladoc.out
exit $EXIT_CODE
fi
-cd ..
echo "============ Checking scala suffixes ============"
diff --git a/tools/maven/scalastyle-config.xml b/tools/maven/scalastyle-config.xml
deleted file mode 100644
index 53b10529d16..00000000000
--- a/tools/maven/scalastyle-config.xml
+++ /dev/null
@@ -1,146 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<!-- NOTE: This was taken and adapted from Apache Spark. -->
-
-<!-- If you wish to turn off checking for a section of code, you can put a comment in the source
- before and after the section, with the following syntax: -->
-<!-- // scalastyle:off -->
-<!-- ... -->
-<!-- // naughty stuff -->
-<!-- ... -->
-<!-- // scalastyle:on -->
-
-<scalastyle>
- <name>Scalastyle standard configuration</name>
- <check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true" />
- <!-- <check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="maxFileLength"><![CDATA[800]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <check level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
- <parameters>
- <parameter name="header"><![CDATA[/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */]]></parameter>
- </parameters>
- </check>
- <check level="error" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true" />
- <check level="error" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="false" />
- <check level="error" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true" />
- <check level="error" class="org.scalastyle.file.FileLineLengthChecker" enabled="true">
- <parameters>
- <parameter name="maxLineLength"><![CDATA[100]]></parameter>
- <parameter name="tabSize"><![CDATA[2]]></parameter>
- <parameter name="ignoreImports">true</parameter>
- </parameters>
- </check>
- <check level="error" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true">
- <parameters>
- <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
- </parameters>
- </check>
- <check level="error" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true">
- <parameters>
- <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
- </parameters>
- </check>
- <check level="error" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true">
- <parameters>
- <parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter>
- </parameters>
- </check>
- <check level="error" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="false" />
- <!-- <check level="error" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <check level="error" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
- <parameters>
- <parameter name="maxParameters"><![CDATA[20]]></parameter>
- </parameters>
- </check>
- <!-- <check level="error" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <check level="error" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="false" />
- <check level="error" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="false" />
- <!-- <check level="error" class="org.scalastyle.scalariform.ReturnChecker" enabled="true" /> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.NullChecker" enabled="true" /> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true" /> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true" /> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true" /> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true" /> -->
- <!-- <check level="error" class="org.scalastyle.file.RegexChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="regex"><![CDATA[println]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="maxTypes"><![CDATA[30]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="maximum"><![CDATA[10]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <check level="error" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true" />
- <check level="error" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="false" />
- <check level="error" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true">
- <parameters>
- <parameter name="singleLineAllowed"><![CDATA[true]]></parameter>
- <parameter name="doubleLineAllowed"><![CDATA[true]]></parameter>
- </parameters>
- </check>
- <!-- <check level="error" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="maxLength"><![CDATA[50]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true"> -->
- <!-- <parameters> -->
- <!-- <parameter name="maxMethods"><![CDATA[30]]></parameter> -->
- <!-- </parameters> -->
- <!-- </check> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true" /> -->
- <check level="error" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true" />
- <check level="error" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false" />
-</scalastyle>
[flink] 02/05: [FLINK-27232][build] Explicitly configure scalafmt in each module
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 01ca3f8939ce53d24f599017b90b034a638e54ed
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Apr 14 14:08:46 2022 +0200
[FLINK-27232][build] Explicitly configure scalafmt in each module
---
.../flink-hadoop-compatibility/pom.xml | 16 +++++++
flink-connectors/flink-hcatalog/pom.xml | 16 +++++++
.../flink-end-to-end-tests-scala/pom.xml | 16 +++++++
.../flink-quickstart-test/pom.xml | 21 ++++++++++
flink-examples/flink-examples-batch/pom.xml | 16 +++++++
flink-examples/flink-examples-streaming/pom.xml | 16 +++++++
flink-examples/flink-examples-table/pom.xml | 16 +++++++
flink-libraries/flink-cep-scala/pom.xml | 17 ++++++++
flink-libraries/flink-gelly-examples/pom.xml | 17 ++++++++
flink-libraries/flink-gelly-scala/pom.xml | 16 +++++++
flink-scala/pom.xml | 17 ++++++++
flink-streaming-scala/pom.xml | 17 ++++++++
flink-table/flink-table-api-scala-bridge/pom.xml | 16 +++++++
flink-table/flink-table-api-scala/pom.xml | 16 +++++++
flink-table/flink-table-planner/pom.xml | 16 +++++++
flink-tests/pom.xml | 17 ++++++++
pom.xml | 49 ++++++++++------------
17 files changed, 287 insertions(+), 28 deletions(-)
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 7ec965570a5..9186f05a5f7 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -100,6 +100,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- activate API compatibility checks -->
<plugin>
<groupId>com.github.siom79.japicmp</groupId>
diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
index fa222d8bb2b..b98817870ee 100644
--- a/flink-connectors/flink-hcatalog/pom.xml
+++ b/flink-connectors/flink-hcatalog/pom.xml
@@ -97,6 +97,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
index 03a94cf5371..2f32bb30b07 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-scala/pom.xml
@@ -82,6 +82,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
diff --git a/flink-end-to-end-tests/flink-quickstart-test/pom.xml b/flink-end-to-end-tests/flink-quickstart-test/pom.xml
index 634c1851fff..1a2e8b6af0f 100644
--- a/flink-end-to-end-tests/flink-quickstart-test/pom.xml
+++ b/flink-end-to-end-tests/flink-quickstart-test/pom.xml
@@ -60,4 +60,25 @@ under the License.
<version>${project.version}</version>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml
index 13eecd44db1..10af26aa70f 100644
--- a/flink-examples/flink-examples-batch/pom.xml
+++ b/flink-examples/flink-examples-batch/pom.xml
@@ -55,6 +55,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Fail compilation on deprecation warnings to prevent from showing users outdated examples. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index 5d1d0dd5ef2..aaee6c1efa8 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -124,6 +124,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Fail compilation on deprecation warnings to prevent from showing users outdated examples. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
index 1c09ec52ad1..cc126ff93f9 100644
--- a/flink-examples/flink-examples-table/pom.xml
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -97,6 +97,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
index 19d9c634179..e35b50f0d69 100644
--- a/flink-libraries/flink-cep-scala/pom.xml
+++ b/flink-libraries/flink-cep-scala/pom.xml
@@ -109,6 +109,23 @@ under the License.
<!-- Not strictly necessary because of the build-helper-maven-plugin below. -->
<sourceDirectory>src/main/scala</sourceDirectory>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
+
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index a5dc0070d3d..72345f9534f 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -121,6 +121,23 @@
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
+
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index be918e10c2f..e930bc4e45e 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -96,6 +96,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index c4de68fb589..418b33399e7 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -139,6 +139,23 @@ under the License.
</executions>
</plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
+
<!-- activate API compatibility checks -->
<plugin>
<groupId>com.github.siom79.japicmp</groupId>
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index d6533c3482a..5126519ea7d 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -135,6 +135,23 @@ under the License.
</executions>
</plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
+
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-table/flink-table-api-scala-bridge/pom.xml b/flink-table/flink-table-api-scala-bridge/pom.xml
index 38381c1ae5a..4a31d9f1a94 100644
--- a/flink-table/flink-table-api-scala-bridge/pom.xml
+++ b/flink-table/flink-table-api-scala-bridge/pom.xml
@@ -77,6 +77,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-table/flink-table-api-scala/pom.xml b/flink-table/flink-table-api-scala/pom.xml
index fde30cccf6b..b2a9248d651 100644
--- a/flink-table/flink-table-api-scala/pom.xml
+++ b/flink-table/flink-table-api-scala/pom.xml
@@ -88,6 +88,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index e059cbdbf00..d16df6271df 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -280,6 +280,22 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 0cfbe29ad93..421ff9113ae 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -279,6 +279,23 @@ under the License.
</executions>
</plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <file>../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ </plugin>
+
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/pom.xml b/pom.xml
index a03c6ba60e2..849829b0b6a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,6 +158,27 @@ under the License.
<japicmp.referenceVersion>1.15.0</japicmp.referenceVersion>
<japicmp.outputDir>tools/japicmp-output</japicmp.outputDir>
<spotless.version>2.13.0</spotless.version>
+ <spotless.scalafmt.version>3.4.3</spotless.scalafmt.version>
+ <spotless.delimiter>package</spotless.delimiter>
+ <spotless.license.header>
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ </spotless.license.header>
<!-- Can be set to any value to reproduce a specific build. -->
<test.randomization.seed/>
@@ -1959,34 +1980,6 @@ under the License.
<removeUnusedImports />
</java>
- <scala>
- <scalafmt>
- <version>3.4.3</version>
- <!-- This file is in the root of the project to make sure IntelliJ picks it up automatically -->
- <file>.scalafmt.conf</file>
- </scalafmt>
- <licenseHeader>
- <content>/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-</content>
- <delimiter>package </delimiter>
- </licenseHeader>
- </scala>
</configuration>
<executions>
<execution>