You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2022/04/12 15:01:40 UTC
[flink] 02/03: [FLINK-26553][build] Format code with Spotless/scalafmt
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 91d81c427aa6312841ca868d54e8ce6ea721cd60
Author: Rufus Refactor <de...@flink.apache.org>
AuthorDate: Tue Apr 12 16:45:42 2022 +0200
[FLINK-26553][build] Format code with Spotless/scalafmt
---
.../scala/hadoop/mapred/HadoopInputFormat.scala | 1 +
.../scala/hadoop/mapred/HadoopOutputFormat.scala | 1 +
.../scala/hadoop/mapreduce/HadoopInputFormat.scala | 2 +-
.../hadoop/mapreduce/HadoopOutputFormat.scala | 2 +-
.../hadoopcompatibility/scala/HadoopInputs.scala | 78 +-
.../scala/ScalaWritableTypeInfoTest.scala | 1 +
.../scala/WordCountMapredITCase.scala | 23 +-
.../scala/WordCountMapreduceITCase.scala | 29 +-
.../flink/hcatalog/scala/HCatInputFormat.scala | 444 ++++--
.../org/apache/flink/tests/scala/ScalaJob.scala | 16 +-
.../test/Elasticsearch7SinkExample.scala | 39 +-
.../flink/examples/scala/clustering/KMeans.scala | 89 +-
.../examples/scala/graph/ConnectedComponents.scala | 81 +-
.../flink/examples/scala/graph/DeltaPageRank.scala | 104 +-
.../flink/examples/scala/graph/EnumTriangles.scala | 85 +-
.../flink/examples/scala/graph/PageRankBasic.scala | 72 +-
.../scala/graph/TransitiveClosureNaive.scala | 67 +-
.../flink/examples/scala/misc/PiEstimation.scala | 15 +-
.../examples/scala/relational/TPCHQuery10.scala | 176 +--
.../examples/scala/relational/TPCHQuery3.scala | 129 +-
.../examples/scala/relational/WebLogAnalysis.scala | 69 +-
.../flink/examples/scala/wordcount/WordCount.scala | 13 +-
.../scala/examples/async/AsyncClient.scala | 5 +-
.../scala/examples/async/AsyncIOExample.scala | 5 +-
.../scala/examples/iteration/IterateExample.scala | 72 +-
.../streaming/scala/examples/join/WindowJoin.scala | 22 +-
.../scala/examples/join/WindowJoinSampleData.scala | 31 +-
.../examples/socket/SocketWindowWordCount.scala | 31 +-
.../GroupedProcessingTimeWindowExample.scala | 11 +-
.../examples/windowing/SessionWindowing.scala | 39 +-
.../examples/windowing/TopSpeedWindowing.scala | 54 +-
.../scala/examples/windowing/WindowWordCount.scala | 44 +-
.../scala/examples/windowing/util/CarSource.scala | 9 +-
.../scala/examples/wordcount/WordCount.scala | 56 +-
.../scala/examples/wordcount/util/CLI.scala | 1 -
.../scala/examples/StreamingExamplesITCase.scala | 98 +-
.../scala/basics/GettingStartedExample.scala | 99 +-
.../examples/scala/basics/StreamSQLExample.scala | 21 +-
.../examples/scala/basics/StreamTableExample.scala | 29 +-
.../scala/basics/WordCountSQLExample.scala | 25 +-
.../scala/org/apache/flink/cep/scala/CEP.scala | 59 +-
.../org/apache/flink/cep/scala/PatternStream.scala | 590 ++++----
.../flink/cep/scala/conditions/Context.scala | 16 +-
.../scala/org/apache/flink/cep/scala/package.scala | 24 +-
.../flink/cep/scala/pattern/GroupPattern.scala | 47 +-
.../apache/flink/cep/scala/pattern/Pattern.scala | 627 ++++----
.../apache/flink/cep/scala/pattern/package.scala | 36 +-
.../cep/scala/CEPScalaAPICompletenessTest.scala | 13 +-
...ernStreamScalaJavaAPIInteroperabilityTest.scala | 64 +-
.../pattern/PatternScalaAPICompletenessTest.scala | 9 +-
.../flink/cep/scala/pattern/PatternTest.scala | 111 +-
.../graph/scala/examples/ConnectedComponents.scala | 101 +-
.../examples/GSASingleSourceShortestPaths.scala | 61 +-
.../scala/examples/SingleSourceShortestPaths.scala | 71 +-
.../apache/flink/graph/scala/EdgesFunction.scala | 9 +-
.../graph/scala/EdgesFunctionWithVertexValue.scala | 11 +-
.../scala/org/apache/flink/graph/scala/Graph.scala | 1428 ++++++++++--------
.../flink/graph/scala/NeighborsFunction.scala | 14 +-
.../scala/NeighborsFunctionWithVertexValue.scala | 23 +-
.../org/apache/flink/graph/scala/package.scala | 10 +-
.../flink/graph/scala/utils/EdgeToTuple3Map.scala | 1 -
.../graph/scala/utils/Tuple2ToVertexMap.scala | 1 -
.../flink/graph/scala/utils/Tuple3ToEdgeMap.scala | 1 -
.../graph/scala/utils/VertexToTuple2Map.scala | 1 -
.../scala/test/GellyScalaAPICompletenessTest.scala | 11 +-
.../flink/graph/scala/test/TestGraphUtils.scala | 53 +-
.../scala/test/operations/DegreesITCase.scala | 30 +-
.../operations/GraphCreationWithCsvITCase.scala | 119 +-
.../test/operations/GraphMutationsITCase.scala | 173 ++-
.../test/operations/GraphOperationsITCase.scala | 217 ++-
.../test/operations/JoinWithEdgesITCase.scala | 92 +-
.../test/operations/JoinWithVerticesITCase.scala | 31 +-
.../scala/test/operations/MapEdgesITCase.scala | 26 +-
.../scala/test/operations/MapVerticesITCase.scala | 23 +-
.../operations/ReduceOnEdgesMethodsITCase.scala | 88 +-
.../operations/ReduceOnNeighborMethodsITCase.scala | 67 +-
.../apache/flink/api/scala/AggregateDataSet.scala | 34 +-
.../apache/flink/api/scala/ClosureCleaner.scala | 197 +--
.../apache/flink/api/scala/CoGroupDataSet.scala | 93 +-
.../org/apache/flink/api/scala/CrossDataSet.scala | 25 +-
.../scala/org/apache/flink/api/scala/DataSet.scala | 1267 ++++++++--------
.../flink/api/scala/ExecutionEnvironment.scala | 495 +++----
.../apache/flink/api/scala/GroupedDataSet.scala | 311 ++--
.../flink/api/scala/PartitionSortedDataSet.scala | 24 +-
.../flink/api/scala/SelectByMaxFunction.scala | 35 +-
.../flink/api/scala/SelectByMinFunction.scala | 31 +-
.../api/scala/UnfinishedCoGroupOperation.scala | 33 +-
.../api/scala/codegen/MacroContextHolder.scala | 9 +-
.../apache/flink/api/scala/codegen/TreeGen.scala | 381 ++---
.../flink/api/scala/codegen/TypeAnalyzer.scala | 214 ++-
.../flink/api/scala/codegen/TypeDescriptors.scala | 34 +-
.../api/scala/codegen/TypeInformationGen.scala | 81 +-
.../acceptPartialFunctions/OnCoGroupDataSet.scala | 39 +-
.../acceptPartialFunctions/OnCrossDataSet.scala | 34 +-
.../impl/acceptPartialFunctions/OnDataSet.scala | 127 +-
.../acceptPartialFunctions/OnGroupedDataSet.scala | 86 +-
.../OnHalfUnfinishedKeyPairOperation.scala | 38 +-
.../OnJoinFunctionAssigner.scala | 34 +-
.../OnUnfinishedKeyPairOperation.scala | 39 +-
.../flink/api/scala/extensions/package.scala | 75 +-
.../org/apache/flink/api/scala/joinDataSet.scala | 108 +-
.../flink/api/scala/metrics/ScalaGauge.scala | 5 +-
.../scala/org/apache/flink/api/scala/package.scala | 58 +-
.../api/scala/typeutils/CaseClassComparator.scala | 25 +-
.../api/scala/typeutils/CaseClassSerializer.scala | 19 +-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 91 +-
.../api/scala/typeutils/EitherSerializer.scala | 8 +-
.../flink/api/scala/typeutils/EitherTypeInfo.scala | 6 +-
.../api/scala/typeutils/EnumValueComparator.scala | 6 +-
.../api/scala/typeutils/EnumValueSerializer.scala | 27 +-
.../api/scala/typeutils/EnumValueTypeInfo.scala | 14 +-
.../api/scala/typeutils/NothingSerializer.scala | 11 +-
.../api/scala/typeutils/OptionSerializer.scala | 15 +-
.../api/scala/typeutils/OptionTypeComparator.scala | 16 +-
.../flink/api/scala/typeutils/OptionTypeInfo.scala | 10 +-
.../scala/typeutils/ScalaCaseClassSerializer.scala | 23 +-
.../typeutils/ScalaEnumSerializerSnapshot.scala | 13 +-
.../api/scala/typeutils/ScalaNothingTypeInfo.scala | 2 +-
.../scala/typeutils/TraversableSerializer.scala | 55 +-
.../api/scala/typeutils/TraversableTypeInfo.scala | 6 +-
.../flink/api/scala/typeutils/TrySerializer.scala | 21 +-
.../flink/api/scala/typeutils/TryTypeInfo.scala | 10 +-
.../flink/api/scala/typeutils/TypeUtils.scala | 4 +-
.../apache/flink/api/scala/typeutils/Types.scala | 477 +++---
.../flink/api/scala/typeutils/UnitSerializer.scala | 15 +-
.../flink/api/scala/typeutils/UnitTypeInfo.scala | 2 +-
.../api/scala/unfinishedKeyPairOperation.scala | 98 +-
.../org/apache/flink/api/scala/utils/package.scala | 117 +-
.../runtime/types/FlinkScalaKryoInstantiator.scala | 30 +-
.../types/JavaIterableWrapperSerializer.scala | 10 +-
.../api/scala/DeltaIterationSanityCheckTest.scala | 94 +-
.../apache/flink/api/scala/MaxByOperatorTest.scala | 54 +-
.../apache/flink/api/scala/MinByOperatorTest.scala | 55 +-
.../flink/api/scala/SelectByFunctionTest.scala | 250 ++--
.../scala/extensions/base/AcceptPFTestBase.scala | 7 +-
.../api/scala/extensions/data/KeyValuePair.scala | 14 +-
.../OnCoGroupDataSetTest.scala | 31 +-
.../OnCrossDataSetTest.scala | 11 +-
.../acceptPartialFunctions/OnDataSetTest.scala | 91 +-
.../OnGroupedDataSetTest.scala | 37 +-
.../OnHalfUnfinishedKeyPairOperationTest.scala | 90 +-
.../OnJoinFunctionAssignerTest.scala | 121 +-
.../OnUnfinishedKeyPairOperationTest.scala | 95 +-
.../SemanticPropertiesTranslationTest.scala | 65 +-
.../api/scala/io/CollectionInputFormatTest.scala | 79 +-
.../flink/api/scala/io/CsvInputFormatTest.scala | 92 +-
.../flink/api/scala/metrics/ScalaGaugeTest.scala | 1 -
.../scala/runtime/EnumValueComparatorTest.scala | 1 -
.../scala/runtime/GenericPairComparatorTest.scala | 17 +-
.../runtime/KryoGenericTypeSerializerTest.scala | 50 +-
.../runtime/ScalaSpecialTypesSerializerTest.scala | 44 +-
.../scala/runtime/TraversableSerializerTest.scala | 45 +-
.../scala/runtime/TupleComparatorILD2Test.scala | 5 +-
.../scala/runtime/TupleComparatorILD3Test.scala | 2 -
.../scala/runtime/TupleComparatorILDC3Test.scala | 4 +-
.../scala/runtime/TupleComparatorILDX1Test.scala | 6 +-
.../scala/runtime/TupleComparatorILDXC2Test.scala | 6 +-
.../scala/runtime/TupleComparatorISD1Test.scala | 3 +-
.../scala/runtime/TupleComparatorISD2Test.scala | 1 -
.../scala/runtime/TupleComparatorISD3Test.scala | 3 +-
.../runtime/TupleSerializerCompatibilityTest.scala | 17 +-
...TupleSerializerCompatibilityTestGenerator.scala | 29 +-
.../api/scala/runtime/TupleSerializerTest.scala | 113 +-
.../runtime/TupleSerializerTestInstance.scala | 22 +-
.../tuple/base/PairComparatorTestBase.scala | 13 +-
.../tuple/base/TupleComparatorTestBase.scala | 6 +-
.../api/scala/types/TypeInformationGenTest.scala | 180 +--
.../scala/typeutils/CaseClassTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/EitherSerializerTest.scala | 11 +-
.../api/scala/typeutils/EitherTypeInfoTest.scala | 9 +-
.../EnumValueSerializerCompatibilityTest.scala | 57 +-
.../scala/typeutils/EnumValueSerializerTest.scala | 10 +-
.../typeutils/EnumValueSerializerUpgradeTest.scala | 41 +-
.../scala/typeutils/EnumValueTypeInfoTest.scala | 9 +-
.../scala/typeutils/InstantiationUtilTest.scala | 17 +-
.../api/scala/typeutils/OptionTypeInfoTest.scala | 6 +-
.../ScalaCaseClassSerializerReflectionTest.scala | 10 +-
.../typeutils/ScalaCaseClassSerializerTest.scala | 8 +-
.../ScalaCaseClassSerializerUpgradeTest.scala | 39 +-
.../scala/typeutils/ScalaNothingTypeInfoTest.scala | 5 +-
.../TraversableSerializerUpgradeTest.scala | 223 ++-
.../scala/typeutils/TraversableTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/TryTypeInfoTest.scala | 5 +-
.../api/scala/typeutils/TypeExtractionTest.scala | 7 +-
.../api/scala/typeutils/TypeInfoFactoryTest.scala | 17 +-
.../api/scala/typeutils/UnitTypeInfoTest.scala | 5 +-
.../streaming/api/scala/AllWindowedStream.scala | 398 ++---
.../streaming/api/scala/AsyncDataStream.scala | 366 ++---
.../api/scala/BroadcastConnectedStream.scala | 60 +-
.../streaming/api/scala/CloseableIterator.scala | 17 +-
.../streaming/api/scala/CoGroupedStreams.scala | 104 +-
.../streaming/api/scala/ConnectedStreams.scala | 266 ++--
.../flink/streaming/api/scala/DataStream.scala | 899 ++++++------
.../streaming/api/scala/DataStreamUtils.scala | 58 +-
.../flink/streaming/api/scala/JoinedStreams.scala | 127 +-
.../flink/streaming/api/scala/KeyedStream.scala | 532 +++----
.../flink/streaming/api/scala/OutputTag.scala | 11 +-
.../api/scala/StreamExecutionEnvironment.scala | 1000 +++++++------
.../flink/streaming/api/scala/WindowedStream.scala | 401 ++---
.../streaming/api/scala/async/AsyncFunction.scala | 56 +-
.../api/scala/async/JavaResultFutureWrapper.scala | 17 +-
.../streaming/api/scala/async/ResultFuture.scala | 40 +-
.../api/scala/async/RichAsyncFunction.scala | 31 +-
.../async/ScalaRichAsyncFunctionWrapper.scala | 15 +-
.../acceptPartialFunctions/OnConnectedStream.scala | 89 +-
.../impl/acceptPartialFunctions/OnDataStream.scala | 75 +-
.../acceptPartialFunctions/OnJoinedStream.scala | 39 +-
.../acceptPartialFunctions/OnKeyedStream.scala | 29 +-
.../acceptPartialFunctions/OnWindowedStream.scala | 49 +-
.../streaming/api/scala/extensions/package.scala | 77 +-
.../api/scala/function/AllWindowFunction.scala | 28 +-
.../scala/function/ProcessAllWindowFunction.scala | 75 +-
.../api/scala/function/ProcessWindowFunction.scala | 88 +-
.../api/scala/function/RichAllWindowFunction.scala | 14 +-
.../function/RichProcessAllWindowFunction.scala | 22 +-
.../scala/function/RichProcessWindowFunction.scala | 25 +-
.../api/scala/function/RichWindowFunction.scala | 21 +-
.../api/scala/function/StatefulFunction.scala | 12 +-
.../api/scala/function/WindowFunction.scala | 40 +-
.../function/util/ScalaAllWindowFunction.scala | 11 +-
.../util/ScalaAllWindowFunctionWrapper.scala | 8 +-
.../util/ScalaProcessWindowFunctionWrapper.scala | 44 +-
.../scala/function/util/ScalaReduceFunction.scala | 9 +-
.../scala/function/util/ScalaWindowFunction.scala | 11 +-
.../function/util/ScalaWindowFunctionWrapper.scala | 10 +-
.../apache/flink/streaming/api/scala/package.scala | 47 +-
.../api/scala/AllWindowTranslationTest.scala | 343 ++---
.../api/scala/AsyncDataStreamITCase.scala | 45 +-
.../streaming/api/scala/BroadcastStateITCase.scala | 17 +-
.../api/scala/CaseClassFieldAccessorTest.scala | 2 +-
.../streaming/api/scala/CoGroupJoinITCase.scala | 201 +--
.../streaming/api/scala/CoGroupedStreamsTest.scala | 9 +-
.../flink/streaming/api/scala/DataStreamTest.scala | 184 +--
.../streaming/api/scala/IntervalJoinITCase.scala | 22 +-
.../streaming/api/scala/JoinedStreamsTest.scala | 9 +-
.../api/scala/OutputFormatTestPrograms.scala | 40 +-
.../ReinterpretDataStreamAsKeyedStreamITCase.scala | 8 +-
.../streaming/api/scala/SideOutputITCase.scala | 124 +-
.../streaming/api/scala/SlotAllocationTest.scala | 38 +-
.../streaming/api/scala/StateTestPrograms.scala | 85 +-
.../api/scala/StreamExecutionEnvironmentTest.scala | 14 +-
.../api/scala/StreamingOperatorsITCase.scala | 9 +-
.../scala/StreamingScalaAPICompletenessTest.scala | 42 +-
.../api/scala/TimeWindowTranslationTest.scala | 21 +-
.../streaming/api/scala/WindowFunctionITCase.scala | 196 ++-
.../streaming/api/scala/WindowReduceITCase.scala | 276 ++--
.../api/scala/WindowTranslationTest.scala | 358 ++---
.../scala/extensions/base/AcceptPFTestBase.scala | 16 +-
.../api/scala/extensions/data/KeyValuePair.scala | 14 +-
.../OnConnectedDataStreamTest.scala | 71 +-
.../acceptPartialFunctions/OnDataStreamTest.scala | 57 +-
.../OnJoinedStreamTest.scala | 45 +-
.../OnKeyedDataStreamTest.scala | 11 +-
.../OnWindowedStreamTest.scala | 11 +-
.../sink/filesystem/StreamingFileSinkTest.scala | 22 +-
.../CheckingIdentityRichAllWindowFunction.scala | 10 +-
...ckingIdentityRichProcessAllWindowFunction.scala | 2 -
...CheckingIdentityRichProcessWindowFunction.scala | 2 -
.../CheckingIdentityRichWindowFunction.scala | 13 +-
.../api/bridge/scala/DataStreamConversions.scala | 115 +-
.../api/bridge/scala/StreamStatementSet.scala | 4 +-
.../api/bridge/scala/StreamTableEnvironment.scala | 1118 +++++++-------
.../table/api/bridge/scala/TableConversions.scala | 121 +-
.../scala/internal/StreamStatementSetImpl.scala | 16 +-
.../internal/StreamTableEnvironmentImpl.scala | 73 +-
.../flink/table/api/bridge/scala/package.scala | 14 +-
.../internal/StreamTableEnvironmentImplTest.scala | 29 +-
.../types/TypeInfoDataTypeConverterScalaTest.scala | 14 +-
.../table/api/ImplicitExpressionConversions.scala | 474 +++---
.../table/api/ImplicitExpressionOperations.scala | 169 +--
.../scala/org/apache/flink/table/api/package.scala | 41 +-
.../api/ExpressionsConsistencyCheckTest.scala | 51 +-
.../flink/table/api/ImplicitConversionsTest.scala | 15 +-
.../expressions/ObjectToExpressionScalaTest.scala | 5 +-
.../extraction/DataTypeExtractorScalaTest.scala | 52 +-
.../TypeInferenceExtractorScalaTest.scala | 66 +-
.../table/planner/calcite/CalciteConfig.scala | 92 +-
.../flink/table/planner/calcite/FlinkContext.scala | 9 +-
.../table/planner/calcite/FlinkContextImpl.scala | 1 -
.../table/planner/calcite/FlinkLocalRef.scala | 29 +-
.../planner/calcite/FlinkLogicalRelFactories.scala | 88 +-
.../table/planner/calcite/FlinkPlannerImpl.scala | 117 +-
.../table/planner/calcite/FlinkRelFactories.scala | 35 +-
.../calcite/FlinkRelOptClusterFactory.scala | 9 +-
.../table/planner/calcite/FlinkTypeFactory.scala | 238 +--
.../planner/calcite/PreValidateReWriter.scala | 214 +--
.../table/planner/codegen/CalcCodeGenerator.scala | 22 +-
.../table/planner/codegen/CodeGenException.scala | 5 +-
.../flink/table/planner/codegen/CodeGenUtils.scala | 386 +++--
.../planner/codegen/CodeGeneratorContext.scala | 521 ++++---
.../planner/codegen/CollectorCodeGenerator.scala | 66 +-
.../planner/codegen/CorrelateCodeGenerator.scala | 29 +-
.../planner/codegen/EqualiserCodeGenerator.scala | 26 +-
.../planner/codegen/ExpandCodeGenerator.scala | 28 +-
.../table/planner/codegen/ExprCodeGenerator.scala | 397 ++---
.../table/planner/codegen/ExpressionReducer.scala | 321 ++--
.../codegen/ExternalGeneratedExpression.scala | 51 +-
.../planner/codegen/FunctionCodeGenerator.scala | 217 +--
.../table/planner/codegen/GenerateUtils.scala | 386 +++--
.../planner/codegen/GeneratedExpression.scala | 69 +-
.../table/planner/codegen/HashCodeGenerator.scala | 74 +-
.../flink/table/planner/codegen/Indenter.scala | 10 +-
.../planner/codegen/InputFormatCodeGenerator.scala | 66 +-
.../table/planner/codegen/JsonGenerateUtils.scala | 75 +-
.../planner/codegen/LongHashJoinGenerator.scala | 99 +-
.../planner/codegen/LookupJoinCodeGenerator.scala | 211 ++-
.../table/planner/codegen/MatchCodeGenerator.scala | 378 ++---
.../codegen/NestedLoopJoinCodeGenerator.scala | 119 +-
.../planner/codegen/OperatorCodeGenerator.scala | 87 +-
.../planner/codegen/ProjectionCodeGenerator.scala | 56 +-
.../table/planner/codegen/SinkCodeGenerator.scala | 38 +-
.../planner/codegen/ValuesCodeGenerator.scala | 18 +-
.../codegen/WatermarkGeneratorCodeGenerator.scala | 37 +-
.../table/planner/codegen/agg/AggCodeGen.scala | 16 +-
.../codegen/agg/AggsHandlerCodeGenerator.scala | 492 +++----
.../codegen/agg/DeclarativeAggCodeGen.scala | 131 +-
.../planner/codegen/agg/DistinctAggCodeGen.scala | 256 ++--
.../planner/codegen/agg/ImperativeAggCodeGen.scala | 145 +-
.../codegen/agg/batch/AggCodeGenHelper.scala | 617 ++++----
.../agg/batch/AggWithoutKeysCodeGenerator.scala | 10 +-
.../codegen/agg/batch/HashAggCodeGenHelper.scala | 270 ++--
.../codegen/agg/batch/HashAggCodeGenerator.scala | 67 +-
.../agg/batch/HashWindowCodeGenerator.scala | 246 ++--
.../codegen/agg/batch/SortAggCodeGenerator.scala | 33 +-
.../agg/batch/SortWindowCodeGenerator.scala | 111 +-
.../codegen/agg/batch/WindowCodeGenerator.scala | 154 +-
.../codegen/calls/BridgingFunctionGenUtil.scala | 203 ++-
.../codegen/calls/BridgingSqlFunctionCallGen.scala | 9 +-
.../planner/codegen/calls/BuiltInMethods.scala | 459 +++---
.../planner/codegen/calls/CallGenerator.scala | 13 +-
.../planner/codegen/calls/ConstantCallGen.scala | 7 +-
.../codegen/calls/CurrentTimePointCallGen.scala | 11 +-
.../table/planner/codegen/calls/DivCallGen.scala | 5 +-
.../planner/codegen/calls/ExtractCallGen.scala | 110 +-
.../planner/codegen/calls/FloorCeilCallGen.scala | 13 +-
.../planner/codegen/calls/FunctionGenerator.scala | 813 ++++-------
.../planner/codegen/calls/HashCodeCallGen.scala | 5 +-
.../table/planner/codegen/calls/IfCallGen.scala | 54 +-
.../planner/codegen/calls/JsonArrayCallGen.scala | 51 +-
.../planner/codegen/calls/JsonObjectCallGen.scala | 75 +-
.../planner/codegen/calls/JsonStringCallGen.scala | 5 +-
.../planner/codegen/calls/JsonValueCallGen.scala | 89 +-
.../table/planner/codegen/calls/LikeCallGen.scala | 29 +-
.../planner/codegen/calls/MethodCallGen.scala | 59 +-
.../table/planner/codegen/calls/NotCallGen.scala | 11 +-
.../table/planner/codegen/calls/PrintCallGen.scala | 7 +-
.../table/planner/codegen/calls/RandCallGen.scala | 16 +-
.../codegen/calls/ScalarFunctionCallGen.scala | 85 +-
.../planner/codegen/calls/ScalarOperatorGens.scala | 839 ++++++-----
.../planner/codegen/calls/SearchOperatorGen.scala | 64 +-
.../planner/codegen/calls/StringCallGen.scala | 350 +++--
.../codegen/calls/TableFunctionCallGen.scala | 56 +-
.../codegen/calls/TimestampDiffCallGen.scala | 20 +-
...ltiFieldRangeBoundComparatorCodeGenerator.scala | 13 +-
.../over/RangeBoundComparatorCodeGenerator.scala | 54 +-
.../codegen/sort/ComparatorCodeGenerator.scala | 35 +-
.../planner/codegen/sort/SortCodeGenerator.scala | 99 +-
.../table/planner/delegation/BatchPlanner.scala | 34 +-
.../table/planner/delegation/PlannerBase.scala | 145 +-
.../table/planner/delegation/StreamPlanner.scala | 68 +-
.../planner/expressions/ExpressionBridge.scala | 5 +-
.../table/planner/expressions/InputTypeSpec.scala | 38 +-
.../planner/expressions/PlannerExpression.scala | 24 +-
.../expressions/PlannerExpressionConverter.scala | 68 +-
.../expressions/PlannerExpressionUtils.scala | 5 +-
.../table/planner/expressions/Reinterpret.scala | 16 +-
.../flink/table/planner/expressions/TreeNode.scala | 75 +-
.../table/planner/expressions/aggregations.scala | 53 +-
.../flink/table/planner/expressions/call.scala | 99 +-
.../table/planner/expressions/collection.scala | 13 +-
.../planner/expressions/fieldExpression.scala | 25 +-
.../flink/table/planner/expressions/literals.scala | 23 +-
.../table/planner/expressions/overOffsets.scala | 1 -
.../flink/table/planner/expressions/subquery.scala | 9 +-
.../flink/table/planner/expressions/symbols.scala | 27 +-
.../flink/table/planner/expressions/time.scala | 175 +--
.../planner/expressions/windowProperties.scala | 1 -
.../planner/functions/utils/AggSqlFunction.scala | 105 +-
.../functions/utils/ScalarSqlFunction.scala | 77 +-
.../planner/functions/utils/TableSqlFunction.scala | 144 +-
.../functions/utils/UserDefinedFunctionUtils.scala | 729 +++++-----
.../flink/table/planner/plan/cost/FlinkCost.scala | 272 ++--
.../table/planner/plan/cost/FlinkCostBase.scala | 13 +-
.../table/planner/plan/cost/FlinkCostFactory.scala | 5 +-
.../planner/plan/cost/FlinkCostFactoryBase.scala | 15 +-
.../planner/plan/logical/MatchRecognize.scala | 23 +-
.../table/planner/plan/logical/groupWindows.scala | 26 +-
.../metadata/AggCallSelectivityEstimator.scala | 177 ++-
.../metadata/FlinkDefaultRelMetadataProvider.scala | 1 -
.../plan/metadata/FlinkRelMdColumnInterval.scala | 462 +++---
.../plan/metadata/FlinkRelMdColumnNullCount.scala | 170 ++-
.../metadata/FlinkRelMdColumnOriginNullCount.scala | 20 +-
.../plan/metadata/FlinkRelMdColumnUniqueness.scala | 232 +--
.../plan/metadata/FlinkRelMdCumulativeCost.scala | 16 +-
.../plan/metadata/FlinkRelMdDistinctRowCount.scala | 156 +-
.../plan/metadata/FlinkRelMdDistribution.scala | 32 +-
.../FlinkRelMdFilteredColumnInterval.scala | 173 ++-
.../metadata/FlinkRelMdModifiedMonotonicity.scala | 130 +-
.../metadata/FlinkRelMdNonCumulativeCost.scala | 10 +-
.../FlinkRelMdPercentageOriginalRows.scala | 29 +-
.../plan/metadata/FlinkRelMdPopulationSize.scala | 152 +-
.../planner/plan/metadata/FlinkRelMdRowCount.scala | 82 +-
.../plan/metadata/FlinkRelMdSelectivity.scala | 74 +-
.../planner/plan/metadata/FlinkRelMdSize.scala | 172 +--
.../plan/metadata/FlinkRelMdUniqueGroups.scala | 115 +-
.../plan/metadata/FlinkRelMdUniqueKeys.scala | 173 ++-
.../plan/metadata/FlinkRelMdUpsertKeys.scala | 102 +-
.../plan/metadata/FlinkRelMdWindowProperties.scala | 122 +-
.../plan/metadata/SelectivityEstimator.scala | 582 ++++----
.../planner/plan/nodes/FlinkConventions.scala | 13 +-
.../table/planner/plan/nodes/FlinkRelNode.scala | 36 +-
.../table/planner/plan/nodes/calcite/Expand.scala | 36 +-
.../planner/plan/nodes/calcite/LegacySink.scala | 27 +-
.../planner/plan/nodes/calcite/LogicalExpand.scala | 9 +-
.../plan/nodes/calcite/LogicalLegacySink.scala | 32 +-
.../planner/plan/nodes/calcite/LogicalRank.scala | 8 +-
.../planner/plan/nodes/calcite/LogicalSink.scala | 12 +-
.../plan/nodes/calcite/LogicalTableAggregate.scala | 11 +-
.../nodes/calcite/LogicalWatermarkAssigner.scala | 10 +-
.../nodes/calcite/LogicalWindowAggregate.scala | 3 +-
.../calcite/LogicalWindowTableAggregate.scala | 16 +-
.../table/planner/plan/nodes/calcite/Rank.scala | 71 +-
.../table/planner/plan/nodes/calcite/Sink.scala | 28 +-
.../plan/nodes/calcite/TableAggregate.scala | 25 +-
.../plan/nodes/calcite/WatermarkAssigner.scala | 35 +-
.../plan/nodes/calcite/WindowAggregate.scala | 45 +-
.../plan/nodes/calcite/WindowTableAggregate.scala | 23 +-
.../planner/plan/nodes/common/CommonCalc.scala | 44 +-
.../nodes/common/CommonIntermediateTableScan.scala | 9 +-
.../common/CommonPhysicalWindowTableFunction.scala | 10 +-
.../plan/nodes/logical/FlinkLogicalAggregate.scala | 37 +-
.../plan/nodes/logical/FlinkLogicalCalc.scala | 21 +-
.../plan/nodes/logical/FlinkLogicalCorrelate.scala | 15 +-
.../logical/FlinkLogicalDataStreamTableScan.scala | 34 +-
.../nodes/logical/FlinkLogicalDistribution.scala | 52 +-
.../plan/nodes/logical/FlinkLogicalExpand.scala | 19 +-
.../FlinkLogicalIntermediateTableScan.scala | 28 +-
.../plan/nodes/logical/FlinkLogicalIntersect.scala | 11 +-
.../plan/nodes/logical/FlinkLogicalJoin.scala | 22 +-
.../nodes/logical/FlinkLogicalLegacySink.scala | 25 +-
.../FlinkLogicalLegacyTableSourceScan.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalMatch.scala | 12 +-
.../plan/nodes/logical/FlinkLogicalMinus.scala | 7 +-
.../nodes/logical/FlinkLogicalOverAggregate.scala | 43 +-
.../plan/nodes/logical/FlinkLogicalRank.scala | 31 +-
.../plan/nodes/logical/FlinkLogicalRel.scala | 9 +-
.../plan/nodes/logical/FlinkLogicalSink.scala | 7 +-
.../plan/nodes/logical/FlinkLogicalSnapshot.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalSort.scala | 47 +-
.../nodes/logical/FlinkLogicalTableAggregate.scala | 16 +-
.../logical/FlinkLogicalTableFunctionScan.scala | 9 +-
.../logical/FlinkLogicalTableSourceScan.scala | 33 +-
.../plan/nodes/logical/FlinkLogicalUnion.scala | 13 +-
.../plan/nodes/logical/FlinkLogicalValues.scala | 20 +-
.../logical/FlinkLogicalWatermarkAssigner.scala | 28 +-
.../logical/FlinkLogicalWindowAggregate.scala | 3 +-
.../logical/FlinkLogicalWindowTableAggregate.scala | 7 +-
.../plan/nodes/physical/FlinkPhysicalRel.scala | 24 +-
.../batch/BatchPhysicalBoundedStreamScan.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalCalc.scala | 7 +-
.../physical/batch/BatchPhysicalCalcBase.scala | 13 +-
.../physical/batch/BatchPhysicalCorrelate.scala | 22 +-
.../batch/BatchPhysicalCorrelateBase.scala | 42 +-
.../physical/batch/BatchPhysicalExchange.scala | 16 +-
.../nodes/physical/batch/BatchPhysicalExpand.scala | 7 +-
.../batch/BatchPhysicalGroupAggregateBase.scala | 21 +-
.../batch/BatchPhysicalHashAggregate.scala | 44 +-
.../batch/BatchPhysicalHashAggregateBase.scala | 11 +-
.../physical/batch/BatchPhysicalHashJoin.scala | 21 +-
.../batch/BatchPhysicalHashWindowAggregate.scala | 12 +-
.../BatchPhysicalHashWindowAggregateBase.scala | 7 +-
.../batch/BatchPhysicalIntermediateTableScan.scala | 7 +-
.../physical/batch/BatchPhysicalJoinBase.scala | 134 +-
.../physical/batch/BatchPhysicalLegacySink.scala | 6 +-
.../batch/BatchPhysicalLegacyTableSourceScan.scala | 7 +-
.../nodes/physical/batch/BatchPhysicalLimit.scala | 11 +-
.../batch/BatchPhysicalLocalHashAggregate.scala | 35 +-
.../BatchPhysicalLocalHashWindowAggregate.scala | 9 +-
.../batch/BatchPhysicalLocalSortAggregate.scala | 38 +-
.../BatchPhysicalLocalSortWindowAggregate.scala | 5 +-
.../physical/batch/BatchPhysicalLookupJoin.scala | 7 +-
.../batch/BatchPhysicalNestedLoopJoin.scala | 21 +-
.../batch/BatchPhysicalOverAggregate.scala | 5 +-
.../batch/BatchPhysicalOverAggregateBase.scala | 72 +-
.../physical/batch/BatchPhysicalPythonCalc.scala | 16 +-
.../batch/BatchPhysicalPythonCorrelate.scala | 18 +-
.../batch/BatchPhysicalPythonGroupAggregate.scala | 44 +-
.../BatchPhysicalPythonGroupWindowAggregate.scala | 9 +-
.../batch/BatchPhysicalPythonOverAggregate.scala | 3 +-
.../nodes/physical/batch/BatchPhysicalRank.scala | 18 +-
.../nodes/physical/batch/BatchPhysicalRel.scala | 10 +-
.../nodes/physical/batch/BatchPhysicalSink.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalSort.scala | 13 +-
.../batch/BatchPhysicalSortAggregate.scala | 44 +-
.../batch/BatchPhysicalSortAggregateBase.scala | 5 +-
.../physical/batch/BatchPhysicalSortLimit.scala | 21 +-
.../batch/BatchPhysicalSortMergeJoin.scala | 58 +-
.../batch/BatchPhysicalSortWindowAggregate.scala | 5 +-
.../BatchPhysicalSortWindowAggregateBase.scala | 3 +-
.../batch/BatchPhysicalTableSourceScan.scala | 11 +-
.../nodes/physical/batch/BatchPhysicalUnion.scala | 21 +-
.../nodes/physical/batch/BatchPhysicalValues.scala | 10 +-
.../batch/BatchPhysicalWindowAggregateBase.scala | 41 +-
.../batch/BatchPhysicalWindowTableFunction.scala | 23 +-
.../physical/common/CommonPhysicalExchange.scala | 20 +-
.../nodes/physical/common/CommonPhysicalJoin.scala | 43 +-
.../CommonPhysicalLegacyTableSourceScan.scala | 10 +-
.../physical/common/CommonPhysicalLookupJoin.scala | 193 ++-
.../common/CommonPhysicalTableSourceScan.scala | 11 +-
.../nodes/physical/stream/StreamPhysicalCalc.scala | 9 +-
.../physical/stream/StreamPhysicalCalcBase.scala | 7 +-
.../stream/StreamPhysicalChangelogNormalize.scala | 10 +-
.../physical/stream/StreamPhysicalCorrelate.scala | 22 +-
.../stream/StreamPhysicalCorrelateBase.scala | 30 +-
.../stream/StreamPhysicalDataStreamScan.scala | 13 +-
.../stream/StreamPhysicalDeduplicate.scala | 12 +-
.../stream/StreamPhysicalDropUpdateBefore.scala | 24 +-
.../physical/stream/StreamPhysicalExchange.scala | 7 +-
.../physical/stream/StreamPhysicalExpand.scala | 9 +-
.../StreamPhysicalGlobalGroupAggregate.scala | 34 +-
.../StreamPhysicalGlobalWindowAggregate.scala | 48 +-
.../stream/StreamPhysicalGroupAggregate.scala | 22 +-
.../stream/StreamPhysicalGroupAggregateBase.scala | 45 +-
.../stream/StreamPhysicalGroupTableAggregate.scala | 8 +-
.../StreamPhysicalGroupTableAggregateBase.scala | 27 +-
.../StreamPhysicalGroupWindowAggregate.scala | 9 +-
.../StreamPhysicalGroupWindowAggregateBase.scala | 39 +-
.../StreamPhysicalGroupWindowTableAggregate.scala | 5 +-
.../StreamPhysicalIncrementalGroupAggregate.scala | 37 +-
.../StreamPhysicalIntermediateTableScan.scala | 7 +-
.../stream/StreamPhysicalIntervalJoin.scala | 63 +-
.../nodes/physical/stream/StreamPhysicalJoin.scala | 59 +-
.../physical/stream/StreamPhysicalLegacySink.scala | 6 +-
.../StreamPhysicalLegacyTableSourceScan.scala | 5 +-
.../physical/stream/StreamPhysicalLimit.scala | 18 +-
.../stream/StreamPhysicalLocalGroupAggregate.scala | 28 +-
.../StreamPhysicalLocalWindowAggregate.scala | 46 +-
.../physical/stream/StreamPhysicalLookupJoin.scala | 6 +-
.../physical/stream/StreamPhysicalMatch.scala | 45 +-
.../stream/StreamPhysicalMiniBatchAssigner.scala | 11 +-
.../stream/StreamPhysicalOverAggregate.scala | 15 +-
.../stream/StreamPhysicalOverAggregateBase.scala | 32 +-
.../physical/stream/StreamPhysicalPythonCalc.scala | 16 +-
.../stream/StreamPhysicalPythonCorrelate.scala | 18 +-
.../StreamPhysicalPythonGroupAggregate.scala | 29 +-
.../StreamPhysicalPythonGroupTableAggregate.scala | 8 +-
.../StreamPhysicalPythonGroupWindowAggregate.scala | 9 +-
.../stream/StreamPhysicalPythonOverAggregate.scala | 15 +-
.../nodes/physical/stream/StreamPhysicalRank.scala | 7 +-
.../nodes/physical/stream/StreamPhysicalRel.scala | 9 +-
.../nodes/physical/stream/StreamPhysicalSink.scala | 19 +-
.../nodes/physical/stream/StreamPhysicalSort.scala | 25 +-
.../physical/stream/StreamPhysicalSortLimit.scala | 20 +-
.../stream/StreamPhysicalTableSourceScan.scala | 1 -
.../stream/StreamPhysicalTemporalJoin.scala | 67 +-
.../stream/StreamPhysicalTemporalSort.scala | 15 +-
.../physical/stream/StreamPhysicalUnion.scala | 9 +-
.../physical/stream/StreamPhysicalValues.scala | 7 +-
.../stream/StreamPhysicalWatermarkAssigner.scala | 25 +-
.../stream/StreamPhysicalWindowAggregate.scala | 28 +-
.../stream/StreamPhysicalWindowDeduplicate.scala | 11 +-
.../physical/stream/StreamPhysicalWindowJoin.scala | 36 +-
.../physical/stream/StreamPhysicalWindowRank.scala | 6 +-
.../stream/StreamPhysicalWindowTableFunction.scala | 16 +-
.../BatchCommonSubGraphBasedOptimizer.scala | 59 +-
.../optimize/CommonSubGraphBasedOptimizer.scala | 105 +-
.../table/planner/plan/optimize/Optimizer.scala | 21 +-
.../table/planner/plan/optimize/RelNodeBlock.scala | 245 ++--
.../StreamCommonSubGraphBasedOptimizer.scala | 144 +-
.../optimize/program/BatchOptimizeContext.scala | 9 +-
.../plan/optimize/program/FlinkBatchProgram.scala | 209 ++-
.../optimize/program/FlinkChainedProgram.scala | 87 +-
.../FlinkChangelogModeInferenceProgram.scala | 710 ++++-----
.../optimize/program/FlinkDecorrelateProgram.scala | 29 +-
.../plan/optimize/program/FlinkGroupProgram.scala | 30 +-
.../plan/optimize/program/FlinkHepProgram.scala | 37 +-
.../optimize/program/FlinkHepRuleSetProgram.scala | 109 +-
.../FlinkMiniBatchIntervalTraitInitProgram.scala | 9 +-
.../optimize/program/FlinkOptimizeContext.scala | 15 +-
.../optimize/program/FlinkOptimizeProgram.scala | 16 +-
.../program/FlinkRelTimeIndicatorProgram.scala | 10 +-
.../optimize/program/FlinkRuleSetProgram.scala | 36 +-
.../plan/optimize/program/FlinkStreamProgram.scala | 219 ++-
.../optimize/program/FlinkVolcanoProgram.scala | 39 +-
.../optimize/program/StreamOptimizeContext.scala | 13 +-
.../table/planner/plan/reuse/SubplanReuser.scala | 144 +-
.../planner/plan/rules/FlinkBatchRuleSets.scala | 150 +-
.../planner/plan/rules/FlinkStreamRuleSets.scala | 206 ++-
.../rules/common/CommonTemporalTableJoinRule.scala | 29 +-
.../logical/AggregateReduceGroupingRule.scala | 48 +-
.../logical/BatchLogicalWindowAggregateRule.scala | 20 +-
.../plan/rules/logical/CalcRankTransposeRule.scala | 44 +-
.../rules/logical/CalcSnapshotTransposeRule.scala | 14 +-
.../ConstantRankNumberColumnRemoveRule.scala | 9 +-
.../rules/logical/ConvertToNotInOrInRule.scala | 91 +-
.../rules/logical/CorrelateSortToRankRule.scala | 58 +-
.../rules/logical/DecomposeGroupingSetsRule.scala | 333 ++---
.../logical/EnumerableToLogicalTableScan.scala | 19 +-
.../plan/rules/logical/FlinkCalcMergeRule.scala | 63 +-
.../plan/rules/logical/FlinkLimit0RemoveRule.scala | 12 +-
.../plan/rules/logical/FlinkLogicalRankRule.scala | 60 +-
.../plan/rules/logical/FlinkPruneEmptyRules.scala | 33 +-
.../rules/logical/FlinkRewriteSubQueryRule.scala | 69 +-
.../rules/logical/FlinkSubQueryRemoveRule.scala | 85 +-
.../JoinConditionEqualityTransferRule.scala | 91 +-
.../logical/JoinConditionTypeCoerceRule.scala | 52 +-
.../JoinDependentConditionDerivationRule.scala | 127 +-
.../rules/logical/JoinDeriveNullFilterRule.scala | 50 +-
...relateToJoinFromTemporalTableFunctionRule.scala | 102 +-
...gicalCorrelateToJoinFromTemporalTableRule.scala | 150 +-
.../plan/rules/logical/LogicalUnnestRule.scala | 36 +-
.../logical/LogicalWindowAggregateRuleBase.scala | 196 ++-
.../logical/ProjectSemiAntiJoinTransposeRule.scala | 46 +-
.../rules/logical/PruneAggregateCallRule.scala | 78 +-
.../PushFilterIntoLegacyTableSourceScanRule.scala | 38 +-
.../PushLimitIntoLegacyTableSourceScanRule.scala | 61 +-
...ushPartitionIntoLegacyTableSourceScanRule.scala | 87 +-
.../PushProjectIntoLegacyTableSourceScanRule.scala | 69 +-
.../plan/rules/logical/PythonCalcSplitRule.scala | 288 ++--
.../logical/ReplaceIntersectWithSemiJoinRule.scala | 22 +-
.../logical/ReplaceMinusWithAntiJoinRule.scala | 22 +-
.../rules/logical/RewriteIntersectAllRule.scala | 147 +-
.../plan/rules/logical/RewriteMinusAllRule.scala | 107 +-
.../logical/RewriteMultiJoinConditionRule.scala | 53 +-
.../logical/SimplifyFilterConditionRule.scala | 36 +-
.../rules/logical/SimplifyJoinConditionRule.scala | 17 +-
.../plan/rules/logical/SplitAggregateRule.scala | 462 +++---
.../SplitPythonConditionFromCorrelateRule.scala | 45 +-
.../logical/SplitPythonConditionFromJoinRule.scala | 27 +-
.../logical/StreamLogicalWindowAggregateRule.scala | 27 +-
.../TemporalJoinRewriteWithUniqueKeyRule.scala | 79 +-
.../WindowAggregateReduceFunctionsRule.scala | 19 +-
.../rules/logical/WindowGroupReorderRule.scala | 57 +-
.../plan/rules/logical/WindowPropertiesRule.scala | 103 +-
.../rules/physical/FlinkExpandConversionRule.scala | 21 +-
.../physical/batch/BatchPhysicalAggRuleBase.scala | 48 +-
.../batch/BatchPhysicalBoundedStreamScanRule.scala | 9 +-
.../physical/batch/BatchPhysicalCalcRule.scala | 15 +-
...atchPhysicalConstantTableFunctionScanRule.scala | 32 +-
.../batch/BatchPhysicalCorrelateRule.scala | 14 +-
.../batch/BatchPhysicalDistributionRule.scala | 31 +-
.../physical/batch/BatchPhysicalExpandRule.scala | 5 +-
.../physical/batch/BatchPhysicalHashAggRule.scala | 113 +-
.../physical/batch/BatchPhysicalHashJoinRule.scala | 64 +-
.../BatchPhysicalIntermediateTableScanRule.scala | 1 -
.../physical/batch/BatchPhysicalJoinRuleBase.scala | 38 +-
.../batch/BatchPhysicalLegacySinkRule.scala | 25 +-
.../BatchPhysicalLegacyTableSourceScanRule.scala | 1 -
.../physical/batch/BatchPhysicalLimitRule.scala | 34 +-
.../batch/BatchPhysicalLookupJoinRule.scala | 16 +-
.../batch/BatchPhysicalNestedLoopJoinRule.scala | 11 +-
.../batch/BatchPhysicalOverAggregateRule.scala | 65 +-
.../batch/BatchPhysicalPythonCalcRule.scala | 19 +-
.../physical/batch/BatchPhysicalRankRule.scala | 24 +-
.../batch/BatchPhysicalSingleRowJoinRule.scala | 22 +-
.../physical/batch/BatchPhysicalSinkRule.scala | 35 +-
.../physical/batch/BatchPhysicalSortAggRule.scala | 97 +-
.../batch/BatchPhysicalSortLimitRule.scala | 36 +-
.../batch/BatchPhysicalSortMergeJoinRule.scala | 73 +-
.../physical/batch/BatchPhysicalSortRule.scala | 31 +-
.../batch/BatchPhysicalTableSourceScanRule.scala | 5 +-
.../physical/batch/BatchPhysicalUnionRule.scala | 12 +-
.../physical/batch/BatchPhysicalValuesRule.scala | 22 +-
.../batch/BatchPhysicalWindowAggregateRule.scala | 99 +-
.../BatchPhysicalWindowTableFunctionRule.scala | 16 +-
.../physical/batch/EnforceLocalAggRuleBase.scala | 22 +-
.../physical/batch/EnforceLocalHashAggRule.scala | 15 +-
.../physical/batch/EnforceLocalSortAggRule.scala | 73 +-
.../batch/RemoveRedundantLocalHashAggRule.scala | 16 +-
.../batch/RemoveRedundantLocalRankRule.scala | 30 +-
.../batch/RemoveRedundantLocalSortAggRule.scala | 46 +-
.../physical/common/CommonLookupJoinRule.scala | 40 +-
.../ExpandWindowTableFunctionTransposeRule.scala | 101 +-
.../physical/stream/IncrementalAggregateRule.scala | 53 +-
.../stream/MiniBatchIntervalInferRule.scala | 113 +-
...indowTableFunctionIntoWindowAggregateRule.scala | 46 +-
.../physical/stream/StreamPhysicalCalcRule.scala | 12 +-
...reamPhysicalConstantTableFunctionScanRule.scala | 32 +-
.../stream/StreamPhysicalCorrelateRule.scala | 13 +-
.../stream/StreamPhysicalDataStreamScanRule.scala | 5 +-
.../stream/StreamPhysicalDeduplicateRule.scala | 54 +-
.../physical/stream/StreamPhysicalExpandRule.scala | 5 +-
.../stream/StreamPhysicalGroupAggregateRule.scala | 8 +-
.../StreamPhysicalGroupTableAggregateRule.scala | 7 +-
.../StreamPhysicalGroupWindowAggregateRule.scala | 5 +-
...reamPhysicalGroupWindowTableAggregateRule.scala | 7 +-
.../StreamPhysicalIntermediateTableScanRule.scala | 1 -
.../stream/StreamPhysicalIntervalJoinRule.scala | 40 +-
.../physical/stream/StreamPhysicalJoinRule.scala | 14 +-
.../stream/StreamPhysicalJoinRuleBase.scala | 20 +-
.../stream/StreamPhysicalLegacySinkRule.scala | 23 +-
.../StreamPhysicalLegacyTableSourceScanRule.scala | 1 -
.../physical/stream/StreamPhysicalLimitRule.scala | 9 +-
.../stream/StreamPhysicalLookupJoinRule.scala | 24 +-
.../physical/stream/StreamPhysicalMatchRule.scala | 28 +-
.../stream/StreamPhysicalOverAggregateRule.scala | 7 +-
.../stream/StreamPhysicalPythonCalcRule.scala | 12 +-
.../physical/stream/StreamPhysicalRankRule.scala | 7 +-
.../physical/stream/StreamPhysicalSinkRule.scala | 36 +-
.../stream/StreamPhysicalSortLimitRule.scala | 9 +-
.../physical/stream/StreamPhysicalSortRule.scala | 14 +-
.../stream/StreamPhysicalTableSourceScanRule.scala | 22 +-
.../stream/StreamPhysicalTemporalJoinRule.scala | 5 +-
.../stream/StreamPhysicalTemporalSortRule.scala | 34 +-
.../physical/stream/StreamPhysicalUnionRule.scala | 12 +-
.../physical/stream/StreamPhysicalValuesRule.scala | 11 +-
.../StreamPhysicalWatermarkAssignerRule.scala | 15 +-
.../stream/StreamPhysicalWindowAggregateRule.scala | 69 +-
.../StreamPhysicalWindowDeduplicateRule.scala | 14 +-
.../stream/StreamPhysicalWindowJoinRule.scala | 18 +-
.../stream/StreamPhysicalWindowRankRule.scala | 12 +-
.../StreamPhysicalWindowTableFunctionRule.scala | 16 +-
.../stream/TwoStageOptimizedAggregateRule.scala | 70 +-
.../planner/plan/schema/DataStreamTable.scala | 37 +-
.../schema/DeferredTypeFlinkTableFunction.scala | 23 +-
.../planner/plan/schema/FlinkTableFunction.scala | 21 +-
.../planner/plan/schema/GenericRelDataType.scala | 35 +-
.../planner/plan/schema/IntermediateRelTable.scala | 25 +-
.../plan/schema/LegacyCatalogSourceTable.scala | 122 +-
.../plan/schema/LegacyTableSourceTable.scala | 90 +-
.../planner/plan/schema/TableSourceTable.scala | 70 +-
.../plan/schema/TimeIndicatorRelDataType.scala | 15 +-
.../plan/schema/TypedFlinkTableFunction.scala | 29 +-
.../table/planner/plan/stats/FlinkStatistic.scala | 102 +-
.../table/planner/plan/stats/ValueInterval.scala | 158 +-
.../planner/plan/trait/FlinkRelDistribution.scala | 96 +-
.../plan/trait/MiniBatchIntervalTrait.scala | 13 +-
.../plan/trait/MiniBatchIntervalTraitDef.scala | 1 -
.../planner/plan/trait/ModifyKindSetTrait.scala | 25 +-
.../planner/plan/trait/ModifyKindSetTraitDef.scala | 3 -
.../plan/trait/RelModifiedMonotonicity.scala | 8 +-
.../flink/table/planner/plan/trait/TraitUtil.scala | 42 +-
.../table/planner/plan/trait/UpdateKindTrait.scala | 34 +-
.../planner/plan/trait/UpdateKindTraitDef.scala | 5 +-
.../planner/plan/utils/AggFunctionFactory.scala | 120 +-
.../table/planner/plan/utils/AggregateUtil.scala | 788 +++++-----
.../planner/plan/utils/ChangelogPlanUtils.scala | 37 +-
.../planner/plan/utils/ColumnIntervalUtil.scala | 123 +-
.../table/planner/plan/utils/CorrelateUtil.scala | 69 +-
.../table/planner/plan/utils/ExpandUtil.scala | 170 ++-
.../table/planner/plan/utils/FlinkRelMdUtil.scala | 421 +++---
.../table/planner/plan/utils/FlinkRelOptUtil.scala | 371 ++---
.../table/planner/plan/utils/FlinkRexUtil.scala | 255 ++--
.../planner/plan/utils/IntervalJoinUtil.scala | 219 +--
.../flink/table/planner/plan/utils/JoinUtil.scala | 105 +-
.../flink/table/planner/plan/utils/MatchUtil.scala | 33 +-
.../planner/plan/utils/NestedProjectionUtil.scala | 177 +--
.../planner/plan/utils/OverAggregateUtil.scala | 119 +-
.../table/planner/plan/utils/PartitionPruner.scala | 79 +-
.../table/planner/plan/utils/PythonUtil.scala | 118 +-
.../flink/table/planner/plan/utils/RankUtil.scala | 146 +-
.../table/planner/plan/utils/RelExplainUtil.scala | 813 ++++++-----
.../table/planner/plan/utils/RelShuttles.scala | 71 +-
.../planner/plan/utils/RelTreeWriterImpl.scala | 29 +-
.../planner/plan/utils/RexDefaultVisitor.scala | 7 +-
.../planner/plan/utils/RexNodeExtractor.scala | 254 ++--
.../table/planner/plan/utils/RexNodeRewriter.scala | 32 +-
.../flink/table/planner/plan/utils/ScanUtil.scala | 65 +-
.../planner/plan/utils/SetOpRewriteUtil.scala | 46 +-
.../flink/table/planner/plan/utils/SortUtil.scala | 67 +-
.../planner/plan/utils/TemporalJoinUtil.scala | 150 +-
.../planner/plan/utils/UpdatingPlanChecker.scala | 11 +-
.../planner/plan/utils/WindowEmitStrategy.scala | 65 +-
.../table/planner/plan/utils/WindowJoinUtil.scala | 185 ++-
.../table/planner/plan/utils/WindowUtil.scala | 161 +--
.../table/planner/plan/utils/aggregation.scala | 102 +-
.../table/planner/sinks/CollectTableSink.scala | 14 +-
.../table/planner/sinks/DataStreamTableSink.scala | 35 +-
.../flink/table/planner/sinks/TableSinkUtils.scala | 210 +--
.../table/planner/sources/TableSourceUtil.scala | 195 +--
.../planner/typeutils/LegacyDataViewUtils.scala | 54 +-
.../table/planner/typeutils/TypeCoercion.scala | 199 ++-
.../planner/typeutils/TypeInfoCheckUtils.scala | 74 +-
.../planner/utils/JavaScalaConversionUtil.scala | 7 +-
.../apache/flink/table/planner/utils/Logging.scala | 5 +-
.../table/planner/validate/ValidationResult.scala | 19 +-
.../apache/flink/table/api/ExecuteSqlTest.scala | 46 +-
.../flink/table/api/TableEnvironmentITCase.scala | 120 +-
.../flink/table/api/TableEnvironmentTest.scala | 440 +++---
.../org/apache/flink/table/api/TableITCase.scala | 73 +-
.../apache/flink/table/api/batch/ExplainTest.scala | 11 +-
.../flink/table/api/stream/ExplainTest.scala | 80 +-
.../validation/MatchRecognizeValidationTest.scala | 40 +-
.../sql/validation/OverWindowValidationTest.scala | 9 +-
.../planner/calcite/CalciteConfigBuilderTest.scala | 22 +-
.../table/planner/catalog/CatalogTableITCase.scala | 251 ++--
.../table/planner/catalog/CatalogTableTest.scala | 11 +-
.../table/planner/catalog/CatalogViewITCase.scala | 197 +--
.../planner/codegen/HashCodeGeneratorTest.scala | 35 +-
.../codegen/ProjectionCodeGeneratorTest.scala | 102 +-
.../codegen/WatermarkGeneratorCodeGenTest.scala | 42 +-
.../table/planner/codegen/agg/AggTestBase.scala | 18 +-
.../codegen/agg/AggsHandlerCodeGeneratorTest.scala | 37 +-
.../codegen/agg/batch/AggWithoutKeysTest.scala | 73 +-
.../codegen/agg/batch/BatchAggTestBase.scala | 46 +-
.../agg/batch/HashAggCodeGeneratorTest.scala | 89 +-
.../agg/batch/SortAggCodeGeneratorTest.scala | 96 +-
.../table/planner/expressions/ArrayTypeTest.scala | 53 +-
.../planner/expressions/CompositeAccessTest.scala | 13 +-
.../planner/expressions/DecimalTypeTest.scala | 643 ++-------
.../table/planner/expressions/LiteralTest.scala | 5 +-
.../table/planner/expressions/MapTypeTest.scala | 62 +-
.../expressions/NonDeterministicTests.scala | 58 +-
.../table/planner/expressions/RowTypeTest.scala | 17 +-
.../planner/expressions/ScalarFunctionsTest.scala | 1525 ++++++++------------
.../planner/expressions/ScalarOperatorsTest.scala | 74 +-
.../planner/expressions/SqlExpressionTest.scala | 113 +-
.../planner/expressions/TemporalTypesTest.scala | 638 +++-----
.../expressions/utils/ArrayTypeTestBase.scala | 21 +-
.../expressions/utils/CompositeTypeTestBase.scala | 22 +-
.../expressions/utils/ExpressionTestBase.scala | 59 +-
.../expressions/utils/MapTypeTestBase.scala | 21 +-
.../expressions/utils/RowTypeTestBase.scala | 7 +-
.../utils/ScalarOperatorsTestBase.scala | 60 +-
.../expressions/utils/ScalarTypesTestBase.scala | 21 +-
.../utils/userDefinedScalarFunctions.scala | 6 +-
.../validation/ArrayTypeValidationTest.scala | 5 +-
.../validation/CompositeAccessValidationTest.scala | 3 -
.../validation/MapTypeValidationTest.scala | 1 -
.../validation/RowTypeValidationTest.scala | 1 -
.../validation/ScalarFunctionsValidationTest.scala | 36 +-
.../validation/ScalarOperatorsValidationTest.scala | 19 +-
.../utils/TestCollectionTableFactory.scala | 77 +-
.../planner/match/PatternTranslatorTest.scala | 94 +-
.../planner/match/PatternTranslatorTestBase.scala | 41 +-
.../table/planner/plan/batch/sql/CalcTest.scala | 6 +-
.../plan/batch/sql/CompactManagedTableTest.scala | 5 +-
.../plan/batch/sql/DagOptimizationTest.scala | 39 +-
.../plan/batch/sql/DeadlockBreakupTest.scala | 128 +-
.../planner/plan/batch/sql/LegacyLimitTest.scala | 2 +-
.../planner/plan/batch/sql/LegacySinkTest.scala | 41 +-
.../plan/batch/sql/LegacyTableSourceTest.scala | 82 +-
.../table/planner/plan/batch/sql/LimitTest.scala | 31 +-
.../plan/batch/sql/MultipleInputCreationTest.scala | 93 +-
.../plan/batch/sql/PartitionableSinkTest.scala | 1 -
.../plan/batch/sql/PartitionableSourceTest.scala | 46 +-
.../table/planner/plan/batch/sql/RankTest.scala | 44 +-
.../plan/batch/sql/RemoveCollationTest.scala | 110 +-
.../planner/plan/batch/sql/RemoveShuffleTest.scala | 217 +--
.../planner/plan/batch/sql/SetOperatorsTest.scala | 11 +-
.../planner/plan/batch/sql/SortLimitTest.scala | 5 +-
.../table/planner/plan/batch/sql/SortTest.scala | 27 +-
.../planner/plan/batch/sql/SubplanReuseTest.scala | 144 +-
.../planner/plan/batch/sql/TableScanTest.scala | 200 ++-
.../planner/plan/batch/sql/TableSinkTest.scala | 145 +-
.../planner/plan/batch/sql/TableSourceTest.scala | 132 +-
.../table/planner/plan/batch/sql/UnionTest.scala | 1 -
.../table/planner/plan/batch/sql/ValuesTest.scala | 1 -
.../plan/batch/sql/WindowTableFunctionTest.scala | 33 +-
.../sql/agg/AggregateReduceGroupingTest.scala | 4 +-
.../plan/batch/sql/agg/AggregateTestBase.scala | 169 ++-
.../plan/batch/sql/agg/DistinctAggregateTest.scala | 5 +-
.../plan/batch/sql/agg/GroupWindowTest.scala | 49 +-
.../plan/batch/sql/agg/GroupingSetsTest.scala | 22 +-
.../plan/batch/sql/agg/HashAggregateTest.scala | 8 +-
.../plan/batch/sql/agg/OverAggregateTest.scala | 10 +-
.../plan/batch/sql/agg/SortAggregateTest.scala | 8 +-
.../batch/sql/join/BroadcastHashJoinTest.scala | 5 +-
.../sql/join/BroadcastHashSemiAntiJoinTest.scala | 5 +-
.../planner/plan/batch/sql/join/JoinTestBase.scala | 2 +-
.../plan/batch/sql/join/LookupJoinTest.scala | 87 +-
.../plan/batch/sql/join/NestedLoopJoinTest.scala | 5 +-
.../sql/join/NestedLoopSemiAntiJoinTest.scala | 5 +-
.../plan/batch/sql/join/SemiAntiJoinTest.scala | 17 +-
.../plan/batch/sql/join/SemiAntiJoinTestBase.scala | 3 +-
.../plan/batch/sql/join/ShuffledHashJoinTest.scala | 1 -
.../sql/join/ShuffledHashSemiAntiJoinTest.scala | 1 -
.../plan/batch/sql/join/SingleRowJoinTest.scala | 1 -
.../plan/batch/sql/join/SortMergeJoinTest.scala | 5 +-
.../batch/sql/join/SortMergeSemiAntiJoinTest.scala | 5 +-
.../batch/sql/join/TemporalFunctionJoinTest.scala | 30 +-
.../plan/batch/sql/join/TemporalJoinTest.scala | 94 +-
.../planner/plan/batch/table/AggregateTest.scala | 22 +-
.../table/planner/plan/batch/table/CalcTest.scala | 12 +-
.../plan/batch/table/ColumnFunctionsTest.scala | 8 +-
.../planner/plan/batch/table/CorrelateTest.scala | 32 +-
.../planner/plan/batch/table/GroupWindowTest.scala | 27 +-
.../table/planner/plan/batch/table/JoinTest.scala | 39 +-
.../plan/batch/table/PythonAggregateTest.scala | 7 +-
.../planner/plan/batch/table/PythonCalcTest.scala | 1 -
.../table/PythonGroupWindowAggregateTest.scala | 45 +-
.../table/PythonOverWindowAggregateTest.scala | 44 +-
.../plan/batch/table/SetOperatorsTest.scala | 34 +-
.../table/TemporalTableFunctionJoinTest.scala | 14 +-
.../table/validation/AggregateValidationTest.scala | 35 +-
.../table/validation/CalcValidationTest.scala | 24 +-
.../table/validation/CorrelateValidationTest.scala | 12 +-
.../validation/GroupWindowValidationTest.scala | 31 +-
.../table/validation/JoinValidationTest.scala | 26 +-
.../validation/OverWindowValidationTest.scala | 15 +-
.../validation/SetOperatorsValidationTest.scala | 18 +-
.../table/validation/SortValidationTest.scala | 1 -
.../common/AggregateReduceGroupingTestBase.scala | 158 +-
.../plan/common/DistinctAggregateTestBase.scala | 16 +-
.../planner/plan/common/JoinReorderTestBase.scala | 188 ++-
.../planner/plan/common/PartialInsertTest.scala | 74 +-
.../planner/plan/common/TableFactoryTest.scala | 11 +-
.../table/planner/plan/common/UnnestTestBase.scala | 12 +-
.../planner/plan/common/ViewsExpandingTest.scala | 99 +-
.../table/planner/plan/cost/FlinkCostTest.scala | 36 +-
.../table/planner/plan/hint/OptionsHintTest.scala | 87 +-
.../metadata/AggCallSelectivityEstimatorTest.scala | 367 +++--
.../metadata/FlinkRelMdColumnIntervalTest.scala | 365 +++--
.../metadata/FlinkRelMdColumnNullCountTest.scala | 147 +-
.../FlinkRelMdColumnOriginNullCountTest.scala | 68 +-
.../metadata/FlinkRelMdColumnUniquenessTest.scala | 467 +++---
.../metadata/FlinkRelMdCumulativeCostTest.scala | 29 +-
.../metadata/FlinkRelMdDistinctRowCountTest.scala | 681 +++++----
.../plan/metadata/FlinkRelMdDistributionTest.scala | 21 +-
.../FlinkRelMdFilteredColumnIntervalTest.scala | 253 ++--
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 1199 ++++++++++-----
.../FlinkRelMdModifiedMonotonicityTest.scala | 160 +-
.../metadata/FlinkRelMdNonCumulativeCostTest.scala | 27 +-
.../FlinkRelMdPercentageOriginalRowsTest.scala | 44 +-
.../metadata/FlinkRelMdPopulationSizeTest.scala | 325 +++--
.../plan/metadata/FlinkRelMdRowCollationTest.scala | 94 +-
.../plan/metadata/FlinkRelMdRowCountTest.scala | 197 +--
.../plan/metadata/FlinkRelMdSelectivityTest.scala | 536 ++++---
.../planner/plan/metadata/FlinkRelMdSizeTest.scala | 181 ++-
.../plan/metadata/FlinkRelMdUniqueGroupsTest.scala | 684 +++++----
.../plan/metadata/FlinkRelMdUniqueKeysTest.scala | 166 ++-
.../plan/metadata/FlinkRelMdUpsertKeysTest.scala | 181 ++-
.../metadata/MetadataHandlerConsistencyTest.scala | 95 +-
.../planner/plan/metadata/MetadataTestUtil.scala | 129 +-
.../plan/metadata/SelectivityEstimatorTest.scala | 793 ++++++----
.../planner/plan/nodes/calcite/ExpandTest.scala | 9 +-
.../plan/nodes/calcite/RelNodeTestBase.scala | 14 +-
.../optimize/program/FlinkChainedProgramTest.scala | 36 +-
.../program/FlinkHepRuleSetProgramTest.scala | 44 +-
.../optimize/program/FlinkVolcanoProgramTest.scala | 18 +-
.../logical/AggregateReduceGroupingRuleTest.scala | 9 +-
.../logical/CalcPruneAggregateCallRuleTest.scala | 31 +-
.../CalcPythonCorrelateTransposeRuleTest.scala | 7 +-
.../rules/logical/CalcRankTransposeRuleTest.scala | 29 +-
.../rules/logical/ConvertToNotInOrInRuleTest.scala | 25 +-
.../logical/CorrelateSortToRankRuleTest.scala | 14 +-
.../logical/DecomposeGroupingSetsRuleTest.scala | 5 +-
.../logical/ExpressionReductionRulesTest.scala | 5 +-
...AggregateExpandDistinctAggregatesRuleTest.scala | 6 +-
.../FlinkAggregateJoinTransposeRuleTest.scala | 22 +-
.../logical/FlinkAggregateRemoveRuleTest.scala | 40 +-
.../rules/logical/FlinkCalcMergeRuleTest.scala | 25 +-
.../rules/logical/FlinkFilterJoinRuleTest.scala | 13 +-
.../logical/FlinkJoinPushExpressionsRuleTest.scala | 8 +-
.../logical/FlinkJoinToMultiJoinRuleTest.scala | 8 +-
.../rules/logical/FlinkLimit0RemoveRuleTest.scala | 9 +-
.../FlinkLogicalRankRuleForConstantRangeTest.scala | 5 +-
.../FlinkLogicalRankRuleForRangeEndTest.scala | 5 +-
.../rules/logical/FlinkPruneEmptyRulesTest.scala | 8 +-
.../FlinkSemiAntiJoinFilterTransposeRuleTest.scala | 12 +-
.../FlinkSemiAntiJoinJoinTransposeRuleTest.scala | 14 +-
...FlinkSemiAntiJoinProjectTransposeRuleTest.scala | 15 +-
.../JoinConditionEqualityTransferRuleTest.scala | 5 +-
.../logical/JoinConditionTypeCoerceRuleTest.scala | 8 +-
.../JoinDependentConditionDerivationRuleTest.scala | 15 +-
.../logical/JoinDeriveNullFilterRuleTest.scala | 50 +-
...lCorrelateToJoinFromTemporalTableRuleTest.scala | 108 +-
.../plan/rules/logical/LogicalUnnestRuleTest.scala | 5 +-
.../ProjectPruneAggregateCallRuleTest.scala | 19 +-
.../ProjectSemiAntiJoinTransposeRuleTest.scala | 7 +-
...ojectWindowTableFunctionTransposeRuleTest.scala | 36 +-
.../logical/PruneAggregateCallRuleTestBase.scala | 7 +-
...shFilterIntoLegacyTableSourceScanRuleTest.scala | 16 +-
...ushLimitIntoLegacyTableSourceScanRuleTest.scala | 10 +-
...artitionIntoLegacyTableSourceScanRuleTest.scala | 35 +-
...hProjectIntoLegacyTableSourceScanRuleTest.scala | 15 +-
.../rules/logical/PythonCalcSplitRuleTest.scala | 11 +-
.../logical/PythonCorrelateSplitRuleTest.scala | 7 +-
.../rules/logical/PythonMapMergeRuleTest.scala | 18 +-
.../logical/RankNumberColumnRemoveRuleTest.scala | 5 +-
.../logical/RemoveSingleAggregateRuleTest.scala | 5 +-
.../ReplaceIntersectWithSemiJoinRuleTest.scala | 5 +-
.../logical/ReplaceMinusWithAntiJoinRuleTest.scala | 5 +-
.../logical/RewriteIntersectAllRuleTest.scala | 5 +-
.../rules/logical/RewriteMinusAllRuleTest.scala | 5 +-
.../RewriteMultiJoinConditionRuleTest.scala | 58 +-
.../logical/SimplifyFilterConditionRuleTest.scala | 10 +-
.../logical/SimplifyJoinConditionRuleTest.scala | 4 +-
.../rules/logical/SplitAggregateRuleTest.scala | 20 +-
...SplitPythonConditionFromCorrelateRuleTest.scala | 11 +-
.../SplitPythonConditionFromJoinRuleTest.scala | 11 +-
.../TemporalJoinRewriteWithUniqueKeyRuleTest.scala | 161 ++-
.../rules/logical/WindowGroupReorderRuleTest.scala | 4 +-
.../subquery/FlinkRewriteSubQueryRuleTest.scala | 43 +-
.../logical/subquery/SubQueryAntiJoinTest.scala | 9 +-
.../logical/subquery/SubQuerySemiJoinTest.scala | 17 +-
.../rules/logical/subquery/SubQueryTestBase.scala | 7 +-
.../SubqueryCorrelateVariablesValidationTest.scala | 34 +-
.../batch/EnforceLocalAggRuleTestBase.scala | 2 -
.../batch/EnforceLocalHashAggRuleTest.scala | 46 +-
.../batch/EnforceLocalSortAggRuleTest.scala | 48 +-
.../RemoveRedundantLocalHashAggRuleTest.scala | 23 +-
.../batch/RemoveRedundantLocalRankRuleTest.scala | 4 +-
.../RemoveRedundantLocalSortAggRuleTest.scala | 18 +-
.../stream/ChangelogModeInferenceTest.scala | 277 ++--
...xpandWindowTableFunctionTransposeRuleTest.scala | 76 +-
...signerChangelogNormalizeTransposeRuleTest.scala | 110 +-
.../plan/schema/TimeIndicatorRelDataTypeTest.scala | 5 +-
.../planner/plan/stats/ValueIntervalTest.scala | 24 +-
.../table/planner/plan/stream/sql/CalcTest.scala | 5 +-
.../plan/stream/sql/DagOptimizationTest.scala | 201 ++-
.../planner/plan/stream/sql/DeduplicateTest.scala | 18 +-
.../plan/stream/sql/FilterableSourceTest.scala | 39 +-
.../planner/plan/stream/sql/LegacySinkTest.scala | 67 +-
.../plan/stream/sql/LegacyTableSourceTest.scala | 215 ++-
.../table/planner/plan/stream/sql/LimitTest.scala | 8 +-
.../plan/stream/sql/LimitableSourceTest.scala | 5 +-
.../plan/stream/sql/MatchRecognizeTest.scala | 1 -
.../stream/sql/MiniBatchIntervalInferTest.scala | 236 +--
.../plan/stream/sql/ModifiedMonotonicityTest.scala | 24 +-
.../plan/stream/sql/PartitionableSinkTest.scala | 1 -
.../plan/stream/sql/PartitionableSourceTest.scala | 46 +-
.../table/planner/plan/stream/sql/RankTest.scala | 245 ++--
.../stream/sql/RelTimeIndicatorConverterTest.scala | 30 +-
.../planner/plan/stream/sql/SetOperatorsTest.scala | 4 +-
.../planner/plan/stream/sql/SortLimitTest.scala | 8 +-
.../table/planner/plan/stream/sql/SortTest.scala | 5 +-
.../plan/stream/sql/SourceWatermarkTest.scala | 148 +-
.../planner/plan/stream/sql/SubplanReuseTest.scala | 84 +-
.../planner/plan/stream/sql/TableScanTest.scala | 759 +++++-----
.../planner/plan/stream/sql/TableSinkTest.scala | 803 +++++------
.../planner/plan/stream/sql/TableSourceTest.scala | 3 +-
.../table/planner/plan/stream/sql/UnionTest.scala | 78 +-
.../table/planner/plan/stream/sql/ValuesTest.scala | 1 -
.../plan/stream/sql/WindowDeduplicateTest.scala | 52 +-
.../planner/plan/stream/sql/WindowRankTest.scala | 34 +-
.../plan/stream/sql/WindowTableFunctionTest.scala | 63 +-
.../plan/stream/sql/agg/AggregateTest.scala | 310 ++--
.../stream/sql/agg/DistinctAggregateTest.scala | 11 +-
.../plan/stream/sql/agg/GroupWindowTest.scala | 100 +-
.../plan/stream/sql/agg/GroupingSetsTest.scala | 26 +-
.../stream/sql/agg/IncrementalAggregateTest.scala | 5 +-
.../plan/stream/sql/agg/OverAggregateTest.scala | 17 +-
.../stream/sql/agg/TwoStageAggregateTest.scala | 11 +-
.../plan/stream/sql/agg/WindowAggregateTest.scala | 183 ++-
.../plan/stream/sql/join/IntervalJoinTest.scala | 98 +-
.../planner/plan/stream/sql/join/JoinTest.scala | 292 ++--
.../plan/stream/sql/join/LookupJoinTest.scala | 340 ++---
.../plan/stream/sql/join/SemiAntiJoinTest.scala | 1 -
.../stream/sql/join/TemporalFunctionJoinTest.scala | 35 +-
.../plan/stream/sql/join/TemporalJoinTest.scala | 428 +++---
.../plan/stream/sql/join/WindowJoinTest.scala | 111 +-
.../planner/plan/stream/table/AggregateTest.scala | 36 +-
.../table/planner/plan/stream/table/CalcTest.scala | 40 +-
.../plan/stream/table/ColumnFunctionsTest.scala | 41 +-
.../planner/plan/stream/table/CorrelateTest.scala | 38 +-
.../table/GroupWindowTableAggregateTest.scala | 11 +-
.../plan/stream/table/GroupWindowTest.scala | 170 ++-
.../table/planner/plan/stream/table/JoinTest.scala | 83 +-
.../plan/stream/table/LegacyTableSourceTest.scala | 199 ++-
.../plan/stream/table/OverAggregateTest.scala | 166 ++-
.../plan/stream/table/PythonAggregateTest.scala | 23 +-
.../planner/plan/stream/table/PythonCalcTest.scala | 1 -
.../table/PythonGroupWindowAggregateTest.scala | 77 +-
.../table/PythonOverWindowAggregateTest.scala | 33 +-
.../stream/table/PythonTableAggregateTest.scala | 13 +-
.../plan/stream/table/SetOperatorsTest.scala | 9 +-
.../plan/stream/table/TableAggregateTest.scala | 11 +-
.../plan/stream/table/TableSourceTest.scala | 30 +-
.../table/TemporalTableFunctionJoinTest.scala | 71 +-
.../plan/stream/table/TwoStageAggregateTest.scala | 11 +-
.../table/validation/AggregateValidationTest.scala | 22 +-
.../table/validation/CalcValidationTest.scala | 76 +-
.../table/validation/CorrelateValidationTest.scala | 59 +-
.../GroupWindowTableAggregateValidationTest.scala | 15 +-
.../validation/GroupWindowValidationTest.scala | 97 +-
.../validation/LegacyTableSinkValidationTest.scala | 36 +-
.../validation/OverWindowValidationTest.scala | 63 +-
.../validation/SetOperatorsValidationTest.scala | 5 +-
.../validation/TableAggregateValidationTest.scala | 29 +-
.../TemporalTableJoinValidationTest.scala | 32 +-
.../validation/UnsupportedOpsValidationTest.scala | 1 -
.../plan/trait/FlinkRelDistributionTest.scala | 36 +-
.../plan/utils/ColumnIntervalUtilTest.scala | 39 +-
.../planner/plan/utils/FlinkRelMdUtilTest.scala | 1 -
.../planner/plan/utils/FlinkRelOptUtilTest.scala | 72 +-
.../planner/plan/utils/FlinkRexUtilTest.scala | 412 +++---
.../planner/plan/utils/InputTypeBuilder.scala | 1 -
.../plan/utils/NestedProjectionUtilTest.scala | 106 +-
.../planner/plan/utils/PartitionPrunerTest.scala | 33 +-
.../planner/plan/utils/RexNodeExtractorTest.scala | 241 ++--
.../planner/plan/utils/RexNodeRewriterTest.scala | 65 +-
.../table/planner/plan/utils/RexNodeTestBase.scala | 39 +-
.../plan/utils/TestContextTableFactory.scala | 14 +-
.../table/planner/plan/utils/lookupFunctions.scala | 40 +-
.../flink/table/planner/plan/utils/pojos.scala | 6 +-
.../planner/runtime/FileSystemITCaseBase.scala | 124 +-
.../batch/sql/BatchFileSystemITCaseBase.scala | 19 +-
.../planner/runtime/batch/sql/CalcITCase.scala | 1175 ++++++++-------
.../runtime/batch/sql/CodeSplitITCase.scala | 41 +-
.../runtime/batch/sql/CorrelateITCase.scala | 99 +-
.../runtime/batch/sql/CorrelateITCase2.scala | 66 +-
.../planner/runtime/batch/sql/DecimalITCase.scala | 76 +-
.../batch/sql/FileSystemTestCsvITCase.scala | 5 +-
.../runtime/batch/sql/LegacyLimitITCase.scala | 50 +-
.../batch/sql/LegacyTableSourceITCase.scala | 183 ++-
.../runtime/batch/sql/Limit0RemoveITCase.scala | 1 -
.../planner/runtime/batch/sql/LimitITCase.scala | 3 +-
.../planner/runtime/batch/sql/MiscITCase.scala | 116 +-
.../runtime/batch/sql/MultipleInputITCase.scala | 33 +-
.../runtime/batch/sql/OverAggregateITCase.scala | 810 +++++++----
.../batch/sql/PartitionableSinkITCase.scala | 199 +--
.../batch/sql/PartitionableSourceITCase.scala | 134 +-
.../planner/runtime/batch/sql/RankITCase.scala | 191 ++-
.../runtime/batch/sql/SetOperatorsITCase.scala | 31 +-
.../runtime/batch/sql/SortLimitITCase.scala | 60 +-
.../runtime/batch/sql/TableScanITCase.scala | 22 +-
.../runtime/batch/sql/TableSinkITCase.scala | 51 +-
.../runtime/batch/sql/TableSourceITCase.scala | 253 ++--
.../runtime/batch/sql/TimestampITCase.scala | 204 ++-
.../planner/runtime/batch/sql/UnionITCase.scala | 46 +-
.../planner/runtime/batch/sql/UnnestITCase.scala | 105 +-
.../planner/runtime/batch/sql/ValuesITCase.scala | 1 -
.../batch/sql/WindowTableFunctionITCase.scala | 1019 +++++++++----
.../batch/sql/agg/AggregateITCaseBase.scala | 321 +++-
.../sql/agg/AggregateJoinTransposeITCase.scala | 115 +-
.../sql/agg/AggregateReduceGroupingITCase.scala | 597 +++++---
.../batch/sql/agg/AggregateRemoveITCase.scala | 211 ++-
.../sql/agg/DistinctAggregateITCaseBase.scala | 88 +-
.../runtime/batch/sql/agg/GroupWindowITCase.scala | 332 +++--
.../runtime/batch/sql/agg/GroupingSetsITCase.scala | 332 ++++-
.../runtime/batch/sql/agg/HashAggITCase.scala | 11 +-
.../sql/agg/HashDistinctAggregateITCase.scala | 8 +-
.../batch/sql/agg/PruneAggregateCallITCase.scala | 7 +-
.../runtime/batch/sql/agg/SortAggITCase.scala | 97 +-
.../sql/agg/SortDistinctAggregateITCase.scala | 8 +-
.../runtime/batch/sql/join/InnerJoinITCase.scala | 9 +-
.../sql/join/JoinConditionTypeCoerceITCase.scala | 155 +-
.../runtime/batch/sql/join/JoinITCase.scala | 620 +++++---
.../runtime/batch/sql/join/JoinITCaseHelper.scala | 5 +-
.../batch/sql/join/JoinWithoutKeyITCase.scala | 220 ++-
.../runtime/batch/sql/join/LookupJoinITCase.scala | 132 +-
.../runtime/batch/sql/join/OuterJoinITCase.scala | 245 ++--
.../runtime/batch/sql/join/ScalarQueryITCase.scala | 9 +-
.../runtime/batch/sql/join/SemiJoinITCase.scala | 81 +-
.../runtime/batch/table/AggregationITCase.scala | 201 ++-
.../planner/runtime/batch/table/CalcITCase.scala | 175 ++-
.../runtime/batch/table/CorrelateITCase.scala | 83 +-
.../runtime/batch/table/DecimalITCase.scala | 80 +-
.../runtime/batch/table/GroupWindowITCase.scala | 213 +--
.../planner/runtime/batch/table/JoinITCase.scala | 219 ++-
.../runtime/batch/table/LegacyLimitITCase.scala | 21 +-
.../batch/table/LegacyTableSinkITCase.scala | 144 +-
.../planner/runtime/batch/table/LimitITCase.scala | 2 +-
.../runtime/batch/table/OverAggregateITCase.scala | 399 +++--
.../runtime/batch/table/SetOperatorsITCase.scala | 34 +-
.../planner/runtime/batch/table/SortITCase.scala | 29 +-
.../runtime/batch/table/TableSinkITCase.scala | 529 ++++---
...AbstractTwoInputStreamOperatorWithTTLTest.scala | 26 +-
.../harness/GroupAggregateHarnessTest.scala | 16 +-
.../planner/runtime/harness/HarnessTestBase.scala | 42 +-
.../runtime/harness/OverAggregateHarnessTest.scala | 762 ++++------
.../planner/runtime/harness/RankHarnessTest.scala | 21 +-
.../harness/TableAggregateHarnessTest.scala | 29 +-
.../harness/WindowAggregateHarnessTest.scala | 426 ++++--
...WindowAggregateUseDaylightTimeHarnessTest.scala | 97 +-
.../harness/WindowTableFunctionHarnessTest.scala | 881 +++++++----
.../runtime/stream/FsStreamingSinkITCaseBase.scala | 86 +-
.../runtime/stream/sql/AggregateITCase.scala | 631 ++++----
.../runtime/stream/sql/AggregateRemoveITCase.scala | 226 ++-
.../runtime/stream/sql/AsyncLookupJoinITCase.scala | 112 +-
.../planner/runtime/stream/sql/CalcITCase.scala | 240 +--
.../runtime/stream/sql/ChangelogSourceITCase.scala | 296 ++--
.../runtime/stream/sql/CorrelateITCase.scala | 100 +-
.../runtime/stream/sql/DataStreamScalaITCase.scala | 31 +-
.../runtime/stream/sql/DeduplicateITCase.scala | 136 +-
.../stream/sql/FilterableSourceITCase.scala | 18 +-
.../stream/sql/FsStreamingSinkTestCsvITCase.scala | 5 +-
.../runtime/stream/sql/GroupWindowITCase.scala | 165 ++-
.../runtime/stream/sql/IntervalJoinITCase.scala | 176 ++-
.../planner/runtime/stream/sql/JoinITCase.scala | 397 +++--
.../stream/sql/LegacyTableSourceITCase.scala | 195 ++-
.../runtime/stream/sql/Limit0RemoveITCase.scala | 3 +-
.../planner/runtime/stream/sql/LimitITCase.scala | 15 +-
.../runtime/stream/sql/LimitableSourceITCase.scala | 19 +-
.../runtime/stream/sql/LookupJoinITCase.scala | 176 +--
.../runtime/stream/sql/MatchRecognizeITCase.scala | 112 +-
.../runtime/stream/sql/OverAggregateITCase.scala | 439 +++---
.../stream/sql/PartitionableSourceITCase.scala | 48 +-
.../stream/sql/PruneAggregateCallITCase.scala | 6 +-
.../planner/runtime/stream/sql/RankITCase.scala | 430 +++---
.../stream/sql/SemiAntiJoinStreamITCase.scala | 75 +-
.../runtime/stream/sql/SetOperatorsITCase.scala | 13 +-
.../planner/runtime/stream/sql/SortITCase.scala | 41 +-
.../runtime/stream/sql/SortLimitITCase.scala | 25 +-
.../runtime/stream/sql/SourceWatermarkITCase.scala | 71 +-
.../runtime/stream/sql/SplitAggregateITCase.scala | 312 ++--
.../stream/sql/StreamFileSystemITCaseBase.scala | 24 +-
.../stream/sql/StreamFileSystemTestCsvITCase.scala | 5 +-
.../stream/sql/StreamTableEnvironmentITCase.scala | 93 +-
.../runtime/stream/sql/TableScanITCase.scala | 31 +-
.../runtime/stream/sql/TableSinkITCase.scala | 230 +--
.../runtime/stream/sql/TableSourceITCase.scala | 111 +-
.../runtime/stream/sql/TemporalJoinITCase.scala | 520 +++----
.../runtime/stream/sql/TemporalSortITCase.scala | 88 +-
.../sql/TemporalTableFunctionJoinITCase.scala | 40 +-
.../runtime/stream/sql/TimeAttributeITCase.scala | 75 +-
.../runtime/stream/sql/TimestampITCase.scala | 80 +-
.../planner/runtime/stream/sql/UnnestITCase.scala | 53 +-
.../planner/runtime/stream/sql/ValuesITCase.scala | 5 +-
.../runtime/stream/sql/WindowAggregateITCase.scala | 92 +-
.../stream/sql/WindowDeduplicateITCase.scala | 52 +-
.../stream/sql/WindowDistinctAggregateITCase.scala | 191 ++-
.../runtime/stream/sql/WindowJoinITCase.scala | 88 +-
.../runtime/stream/sql/WindowRankITCase.scala | 91 +-
.../stream/sql/WindowTableFunctionITCase.scala | 60 +-
.../runtime/stream/table/AggregateITCase.scala | 132 +-
.../planner/runtime/stream/table/CalcITCase.scala | 234 +--
.../runtime/stream/table/CorrelateITCase.scala | 77 +-
.../runtime/stream/table/GroupWindowITCase.scala | 101 +-
.../table/GroupWindowTableAggregateITCase.scala | 60 +-
.../planner/runtime/stream/table/JoinITCase.scala | 611 +++++---
.../stream/table/LegacyTableSinkITCase.scala | 295 ++--
.../runtime/stream/table/OverAggregateITCase.scala | 282 ++--
.../runtime/stream/table/RetractionITCase.scala | 44 +-
.../runtime/stream/table/SetOperatorsITCase.scala | 37 +-
.../runtime/stream/table/SubQueryITCase.scala | 26 +-
.../stream/table/TableAggregateITCase.scala | 95 +-
.../runtime/stream/table/TableSinkITCase.scala | 1377 +++++++++---------
.../stream/table/TableToDataStreamITCase.scala | 139 +-
.../planner/runtime/utils/BatchTableEnvUtil.scala | 300 ++--
.../planner/runtime/utils/BatchTestBase.scala | 262 ++--
.../runtime/utils/CollectionBatchExecTable.scala | 41 +-
.../utils/InMemoryLookupableTableSource.scala | 65 +-
.../planner/runtime/utils/SortTestUtils.scala | 52 +-
.../planner/runtime/utils/StreamTestSink.scala | 90 +-
.../planner/runtime/utils/StreamingTestBase.scala | 13 +-
.../runtime/utils/StreamingWithAggTestBase.scala | 10 +-
.../utils/StreamingWithMiniBatchTestBase.scala | 4 +-
.../runtime/utils/StreamingWithStateTestBase.scala | 32 +-
.../table/planner/runtime/utils/TestData.scala | 660 ++++++---
.../table/planner/runtime/utils/TestSinkUtil.scala | 11 +-
.../table/planner/runtime/utils/TimeTestUtil.scala | 17 +-
.../utils/UserDefinedFunctionTestUtils.scala | 36 +-
.../flink/table/planner/utils/AvgAggFunction.scala | 117 +-
.../table/planner/utils/CountAggFunction.scala | 6 +-
.../table/planner/utils/DateTimeTestUtil.scala | 1 -
.../planner/utils/LogicalPlanFormatUtils.scala | 10 +-
.../planner/utils/MemoryTableSourceSinkUtil.scala | 34 +-
.../flink/table/planner/utils/SumAggFunction.scala | 44 +-
.../flink/table/planner/utils/TableTestBase.scala | 692 +++++----
.../utils/TestLegacyLimitableTableSource.scala | 24 +-
.../planner/utils/UserDefinedAggFunctions.scala | 27 +-
.../utils/UserDefinedTableAggFunctions.scala | 26 +-
.../planner/utils/UserDefinedTableFunctions.scala | 40 +-
.../table/planner/utils/testTableSourceSinks.scala | 401 +++--
.../api/scala/actions/CountCollectITCase.scala | 17 +-
.../PartitionOperatorTranslationTest.scala | 33 +-
.../BatchScalaAPICompletenessTest.scala | 252 ++--
.../ScalaAPICompletenessTestBase.scala | 29 +-
.../api/scala/functions/ClosureCleanerITCase.scala | 19 +-
.../scala/io/ScalaCsvReaderWithPOJOITCase.scala | 12 +-
.../manual/MassiveCaseClassSortingITCase.scala | 129 +-
.../api/scala/migration/MigrationTestTypes.scala | 1 -
.../migration/ScalaSerializersMigrationTest.scala | 12 +-
.../StatefulJobSavepointMigrationITCase.scala | 81 +-
...StatefulJobWBroadcastStateMigrationITCase.scala | 124 +-
.../api/scala/operators/AggregateITCase.scala | 17 +-
.../scala/operators/AggregateOperatorTest.scala | 13 +-
.../flink/api/scala/operators/CoGroupITCase.scala | 235 +--
.../api/scala/operators/CoGroupOperatorTest.scala | 61 +-
.../flink/api/scala/operators/CrossITCase.scala | 64 +-
.../flink/api/scala/operators/DistinctITCase.scala | 18 +-
.../api/scala/operators/DistinctOperatorTest.scala | 18 +-
.../flink/api/scala/operators/ExamplesITCase.scala | 23 +-
.../flink/api/scala/operators/FilterITCase.scala | 38 +-
.../flink/api/scala/operators/FirstNITCase.scala | 17 +-
.../api/scala/operators/FirstNOperatorTest.scala | 61 +-
.../flink/api/scala/operators/FlatMapITCase.scala | 71 +-
.../api/scala/operators/GroupCombineITCase.scala | 24 +-
.../api/scala/operators/GroupReduceITCase.scala | 761 +++++-----
.../flink/api/scala/operators/GroupingTest.scala | 34 +-
.../flink/api/scala/operators/JoinITCase.scala | 88 +-
.../api/scala/operators/JoinOperatorTest.scala | 36 +-
.../flink/api/scala/operators/MapITCase.scala | 32 +-
.../api/scala/operators/OuterJoinITCase.scala | 83 +-
.../api/scala/operators/PartitionITCase.scala | 84 +-
.../flink/api/scala/operators/ReduceITCase.scala | 89 +-
.../flink/api/scala/operators/SampleITCase.scala | 9 +-
.../api/scala/operators/SortPartitionITCase.scala | 62 +-
.../api/scala/operators/SumMinMaxITCase.scala | 21 +-
.../flink/api/scala/operators/UnionITCase.scala | 13 +-
.../translation/AggregateTranslationTest.scala | 16 +-
.../CoGroupCustomPartitioningTest.scala | 167 ++-
.../CoGroupGroupSortTranslationTest.scala | 107 +-
...CustomPartitioningGroupingKeySelectorTest.scala | 102 +-
.../CustomPartitioningGroupingPojoTest.scala | 80 +-
.../CustomPartitioningGroupingTupleTest.scala | 184 ++-
.../translation/CustomPartitioningTest.scala | 143 +-
.../DeltaIterationTranslationTest.scala | 52 +-
.../translation/JoinCustomPartitioningTest.scala | 167 ++-
.../translation/PartitioningTestClasses.scala | 3 -
.../translation/ReduceTranslationTest.scala | 63 +-
.../CaseClassNormalizedKeySortingTest.scala | 69 +-
.../scala/runtime/ScalaSpecialTypesITCase.scala | 101 +-
.../flink/api/scala/util/CollectionDataSets.scala | 56 +-
.../flink/api/scala/util/DataSetUtilsITCase.scala | 7 +-
1297 files changed, 60656 insertions(+), 54415 deletions(-)
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala
index 52a6bed4870..ec6796d2e34 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopInputFormat.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.hadoop.mapred
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapred.HadoopInputFormatBase
+
import org.apache.hadoop.mapred.{InputFormat, JobConf}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
index b78665115e0..bfa65445248 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.hadoop.mapred
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapred.HadoopOutputFormatBase
+
import org.apache.hadoop.mapred.{JobConf, OutputCommitter, OutputFormat}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala
index 8efb53de1c0..3fa6666b201 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopInputFormat.scala
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.scala.hadoop.mapreduce
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormatBase
+
import org.apache.hadoop.mapreduce.{InputFormat, Job}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala
index 8095304268b..23c35c8f9f8 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/api/scala/hadoop/mapreduce/HadoopOutputFormat.scala
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.scala.hadoop.mapreduce
import org.apache.flink.annotation.Public
import org.apache.flink.api.java.hadoop.mapreduce.HadoopOutputFormatBase
+
import org.apache.hadoop.mapreduce.{Job, OutputFormat}
@Public
diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala
index a59af64d22c..6275574b8ab 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/scala/org/apache/flink/hadoopcompatibility/scala/HadoopInputs.scala
@@ -15,32 +15,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.hadoopcompatibility.scala
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.scala.hadoop.{mapred, mapreduce}
+
import org.apache.hadoop.fs.{Path => HadoopPath}
-import org.apache.hadoop.mapred.{JobConf, FileInputFormat => MapredFileInputFormat, InputFormat => MapredInputFormat}
+import org.apache.hadoop.mapred.{FileInputFormat => MapredFileInputFormat, InputFormat => MapredInputFormat, JobConf}
+import org.apache.hadoop.mapreduce.{InputFormat => MapreduceInputFormat, Job}
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat => MapreduceFileInputFormat}
-import org.apache.hadoop.mapreduce.{Job, InputFormat => MapreduceInputFormat}
/**
- * HadoopInputs is a utility class to use Apache Hadoop InputFormats with Apache Flink.
- *
- * It provides methods to create Flink InputFormat wrappers for Hadoop
- * [[org.apache.hadoop.mapred.InputFormat]] and [[org.apache.hadoop.mapreduce.InputFormat]].
- *
- * Key value pairs produced by the Hadoop InputFormats are converted into [[Tuple2]] where
- * the first field is the key and the second field is the value.
- *
- */
+ * HadoopInputs is a utility class to use Apache Hadoop InputFormats with Apache Flink.
+ *
+ * It provides methods to create Flink InputFormat wrappers for Hadoop
+ * [[org.apache.hadoop.mapred.InputFormat]] and [[org.apache.hadoop.mapreduce.InputFormat]].
+ *
+ * Key value pairs produced by the Hadoop InputFormats are converted into [[Tuple2]] where the first
+ * field is the key and the second field is the value.
+ */
object HadoopInputs {
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapred.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapred.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapredInputFormat: MapredFileInputFormat[K, V],
key: Class[K],
@@ -55,9 +54,9 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapred.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapred.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapredInputFormat: MapredFileInputFormat[K, V],
key: Class[K],
@@ -68,26 +67,24 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that reads a Hadoop sequence
- * file with the given key and value classes.
- */
- def readSequenceFile[K, V](
- key: Class[K],
- value: Class[V],
- inputPath: String)(implicit tpe: TypeInformation[(K, V)]): mapred.HadoopInputFormat[K, V] = {
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that reads a Hadoop sequence
+ * file with the given key and value classes.
+ */
+ def readSequenceFile[K, V](key: Class[K], value: Class[V], inputPath: String)(implicit
+ tpe: TypeInformation[(K, V)]): mapred.HadoopInputFormat[K, V] = {
readHadoopFile(
new org.apache.hadoop.mapred.SequenceFileInputFormat[K, V],
key,
value,
inputPath
- )
+ )
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapred.InputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapred.InputFormat]].
+ */
def createHadoopInput[K, V](
mapredInputFormat: MapredInputFormat[K, V],
key: Class[K],
@@ -98,9 +95,9 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapreduceInputFormat: MapreduceFileInputFormat[K, V],
key: Class[K],
@@ -115,22 +112,22 @@ object HadoopInputs {
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapreduce.lib.input.FileInputFormat]].
+ */
def readHadoopFile[K, V](
mapreduceInputFormat: MapreduceFileInputFormat[K, V],
key: Class[K],
value: Class[V],
- inputPath: String)(implicit tpe: TypeInformation[(K, V)]): mapreduce.HadoopInputFormat[K, V] =
- {
+ inputPath: String)(implicit
+ tpe: TypeInformation[(K, V)]): mapreduce.HadoopInputFormat[K, V] = {
readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance)
}
/**
- * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
- * [[org.apache.hadoop.mapreduce.InputFormat]].
- */
+ * Creates a Flink [[org.apache.flink.api.common.io.InputFormat]] that wraps the given Hadoop
+ * [[org.apache.hadoop.mapreduce.InputFormat]].
+ */
def createHadoopInput[K, V](
mapreduceInputFormat: MapreduceInputFormat[K, V],
key: Class[K],
@@ -140,4 +137,3 @@ object HadoopInputs {
new mapreduce.HadoopInputFormat[K, V](mapreduceInputFormat, key, value, job)
}
}
-
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala
index df5f36152b6..b54fef4f1a1 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/ScalaWritableTypeInfoTest.scala
@@ -20,6 +20,7 @@ package org.apache.flink.api.hadoopcompatibility.scala
import org.apache.flink.api.java.typeutils.WritableTypeInfo
import org.apache.flink.api.scala._
import org.apache.flink.util.TestLogger
+
import org.apache.hadoop.io.Text
import org.junit.Assert._
import org.junit.Test
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala
index 1bf470c9ab7..ce4115b13a7 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapredITCase.scala
@@ -23,6 +23,7 @@ import org.apache.flink.hadoopcompatibility.scala.HadoopInputs
import org.apache.flink.test.testdata.WordCountData
import org.apache.flink.test.util.{JavaProgramTestBase, TestBaseUtils}
import org.apache.flink.util.OperatingSystem
+
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{LongWritable, Text}
import org.apache.hadoop.mapred.{FileOutputFormat, JobConf, TextInputFormat, TextOutputFormat}
@@ -38,31 +39,34 @@ class WordCountMapredITCase extends JavaProgramTestBase {
Assume.assumeTrue("This test can't run successfully on Windows.", !OperatingSystem.isWindows)
}
- protected override def preSubmit() {
+ override protected def preSubmit() {
textPath = createTempFile("text.txt", WordCountData.TEXT)
resultPath = getTempDirPath("result")
}
- protected override def postSubmit() {
- TestBaseUtils.compareResultsByLinesInMemory(WordCountData.COUNTS,
- resultPath, Array[String](".", "_"))
+ override protected def postSubmit() {
+ TestBaseUtils.compareResultsByLinesInMemory(
+ WordCountData.COUNTS,
+ resultPath,
+ Array[String](".", "_"))
}
- private def internalRun (): Unit = {
+ private def internalRun(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val input =
- env.createInput(HadoopInputs.readHadoopFile(new TextInputFormat, classOf[LongWritable],
- classOf[Text], textPath))
+ env.createInput(
+ HadoopInputs
+ .readHadoopFile(new TextInputFormat, classOf[LongWritable], classOf[Text], textPath))
val counts = input
.map(_._2.toString)
- .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map( (_, 1)))
+ .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map((_, 1)))
.groupBy(0)
.sum(1)
val words = counts
- .map( t => (new Text(t._1), new LongWritable(t._2)) )
+ .map(t => (new Text(t._1), new LongWritable(t._2)))
val hadoopOutputFormat = new HadoopOutputFormat[Text, LongWritable](
new TextOutputFormat[Text, LongWritable],
@@ -81,4 +85,3 @@ class WordCountMapredITCase extends JavaProgramTestBase {
postSubmit()
}
}
-
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala
index d8d0646e362..597e0164259 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/scala/org/apache/flink/api/hadoopcompatibility/scala/WordCountMapreduceITCase.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.hadoopcompatibility.scala
import org.apache.flink.api.scala._
@@ -24,6 +23,7 @@ import org.apache.flink.hadoopcompatibility.scala.HadoopInputs
import org.apache.flink.test.testdata.WordCountData
import org.apache.flink.test.util.{JavaProgramTestBase, TestBaseUtils}
import org.apache.flink.util.OperatingSystem
+
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{LongWritable, Text}
import org.apache.hadoop.mapreduce.Job
@@ -41,14 +41,16 @@ class WordCountMapreduceITCase extends JavaProgramTestBase {
Assume.assumeTrue("This test can't run successfully on Windows.", !OperatingSystem.isWindows)
}
- protected override def preSubmit() {
+ override protected def preSubmit() {
textPath = createTempFile("text.txt", WordCountData.TEXT)
resultPath = getTempDirPath("result")
}
- protected override def postSubmit() {
- TestBaseUtils.compareResultsByLinesInMemory(WordCountData.COUNTS,
- resultPath, Array[String](".", "_"))
+ override protected def postSubmit() {
+ TestBaseUtils.compareResultsByLinesInMemory(
+ WordCountData.COUNTS,
+ resultPath,
+ Array[String](".", "_"))
}
protected def testProgram() {
@@ -56,26 +58,26 @@ class WordCountMapreduceITCase extends JavaProgramTestBase {
postSubmit()
}
- private def internalRun (): Unit = {
+ private def internalRun(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val input =
- env.createInput(HadoopInputs.readHadoopFile(new TextInputFormat, classOf[LongWritable],
- classOf[Text], textPath))
+ env.createInput(
+ HadoopInputs
+ .readHadoopFile(new TextInputFormat, classOf[LongWritable], classOf[Text], textPath))
val counts = input
.map(_._2.toString)
- .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map( (_, 1)))
+ .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty).map((_, 1)))
.groupBy(0)
.sum(1)
val words = counts
- .map( t => (new Text(t._1), new LongWritable(t._2)) )
+ .map(t => (new Text(t._1), new LongWritable(t._2)))
val job = Job.getInstance()
- val hadoopOutputFormat = new HadoopOutputFormat[Text, LongWritable](
- new TextOutputFormat[Text, LongWritable],
- job)
+ val hadoopOutputFormat =
+ new HadoopOutputFormat[Text, LongWritable](new TextOutputFormat[Text, LongWritable], job)
hadoopOutputFormat.getConfiguration.set("mapred.textoutputformat.separator", " ")
FileOutputFormat.setOutputPath(job, new Path(resultPath))
@@ -85,4 +87,3 @@ class WordCountMapreduceITCase extends JavaProgramTestBase {
env.execute("Hadoop Compat WordCount")
}
}
-
diff --git a/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala b/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala
index 64918156e56..d19fd01e78a 100644
--- a/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala
+++ b/flink-connectors/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatInputFormat.scala
@@ -15,28 +15,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.hcatalog.scala
import org.apache.flink.configuration
import org.apache.flink.hcatalog.HCatInputFormatBase
+
import org.apache.hadoop.conf.Configuration
import org.apache.hive.hcatalog.data.HCatRecord
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema
/**
- * A InputFormat to read from HCatalog tables.
- * The InputFormat supports projection (selection and order of fields) and partition filters.
- *
- * Data can be returned as [[HCatRecord]] or Scala tuples.
- * Scala tuples support only up to 22 fields.
+ * A InputFormat to read from HCatalog tables. The InputFormat supports projection (selection and
+ * order of fields) and partition filters.
*
+ * Data can be returned as [[HCatRecord]] or Scala tuples. Scala tuples support only up to 22
+ * fields.
*/
class HCatInputFormat[T](
- database: String,
- table: String,
- config: Configuration
- ) extends HCatInputFormatBase[T](database, table, config) {
+ database: String,
+ table: String,
+ config: Configuration
+) extends HCatInputFormatBase[T](database, table, config) {
def this(database: String, table: String) {
this(database, table, new Configuration)
@@ -57,97 +56,87 @@ class HCatInputFormat[T](
var i: Int = 0
while (i < this.fieldNames.length) {
- val o: AnyRef = record.get(this.fieldNames(i), this.outputSchema)
-
- // partition columns are returned as String
- // Check and convert to actual type.
- this.outputSchema.get(i).getType match {
- case HCatFieldSchema.Type.INT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toInt
- }
- else {
- vals(i) = o.asInstanceOf[Int]
- }
- case HCatFieldSchema.Type.TINYINT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toInt.toByte
- }
- else {
- vals(i) = o.asInstanceOf[Byte]
- }
- case HCatFieldSchema.Type.SMALLINT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toInt.toShort
- }
- else {
- vals(i) = o.asInstanceOf[Short]
- }
- case HCatFieldSchema.Type.BIGINT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toLong
- }
- else {
- vals(i) = o.asInstanceOf[Long]
- }
- case HCatFieldSchema.Type.BOOLEAN =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toBoolean
- }
- else {
- vals(i) = o.asInstanceOf[Boolean]
- }
- case HCatFieldSchema.Type.FLOAT =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toFloat
- }
- else {
- vals(i) = o.asInstanceOf[Float]
- }
- case HCatFieldSchema.Type.DOUBLE =>
- if (o.isInstanceOf[String]) {
- vals(i) = o.asInstanceOf[String].toDouble
- }
- else {
- vals(i) = o.asInstanceOf[Double]
- }
- case HCatFieldSchema.Type.STRING =>
- vals(i) = o
- case HCatFieldSchema.Type.BINARY =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type BINARY.")
- }
- else {
- vals(i) = o.asInstanceOf[Array[Byte]]
- }
- case HCatFieldSchema.Type.ARRAY =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type ARRAY.")
- }
- else {
- vals(i) = o.asInstanceOf[List[Object]]
- }
- case HCatFieldSchema.Type.MAP =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type MAP.")
- }
- else {
- vals(i) = o.asInstanceOf[Map[Object, Object]]
- }
- case HCatFieldSchema.Type.STRUCT =>
- if (o.isInstanceOf[String]) {
- throw new RuntimeException("Cannot handle partition keys of type STRUCT.")
- }
- else {
- vals(i) = o.asInstanceOf[List[Object]]
- }
- case _ =>
- throw new RuntimeException("Invalid type " + this.outputSchema.get(i).getType +
- " encountered.")
- }
+ val o: AnyRef = record.get(this.fieldNames(i), this.outputSchema)
- i += 1
+ // partition columns are returned as String
+ // Check and convert to actual type.
+ this.outputSchema.get(i).getType match {
+ case HCatFieldSchema.Type.INT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toInt
+ } else {
+ vals(i) = o.asInstanceOf[Int]
+ }
+ case HCatFieldSchema.Type.TINYINT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toInt.toByte
+ } else {
+ vals(i) = o.asInstanceOf[Byte]
+ }
+ case HCatFieldSchema.Type.SMALLINT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toInt.toShort
+ } else {
+ vals(i) = o.asInstanceOf[Short]
+ }
+ case HCatFieldSchema.Type.BIGINT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toLong
+ } else {
+ vals(i) = o.asInstanceOf[Long]
+ }
+ case HCatFieldSchema.Type.BOOLEAN =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toBoolean
+ } else {
+ vals(i) = o.asInstanceOf[Boolean]
+ }
+ case HCatFieldSchema.Type.FLOAT =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toFloat
+ } else {
+ vals(i) = o.asInstanceOf[Float]
+ }
+ case HCatFieldSchema.Type.DOUBLE =>
+ if (o.isInstanceOf[String]) {
+ vals(i) = o.asInstanceOf[String].toDouble
+ } else {
+ vals(i) = o.asInstanceOf[Double]
+ }
+ case HCatFieldSchema.Type.STRING =>
+ vals(i) = o
+ case HCatFieldSchema.Type.BINARY =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type BINARY.")
+ } else {
+ vals(i) = o.asInstanceOf[Array[Byte]]
+ }
+ case HCatFieldSchema.Type.ARRAY =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type ARRAY.")
+ } else {
+ vals(i) = o.asInstanceOf[List[Object]]
+ }
+ case HCatFieldSchema.Type.MAP =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type MAP.")
+ } else {
+ vals(i) = o.asInstanceOf[Map[Object, Object]]
+ }
+ case HCatFieldSchema.Type.STRUCT =>
+ if (o.isInstanceOf[String]) {
+ throw new RuntimeException("Cannot handle partition keys of type STRUCT.")
+ } else {
+ vals(i) = o.asInstanceOf[List[Object]]
+ }
+ case _ =>
+ throw new RuntimeException(
+ "Invalid type " + this.outputSchema.get(i).getType +
+ " encountered.")
}
+
+ i += 1
+ }
createScalaTuple(vals)
}
@@ -172,58 +161,247 @@ class HCatInputFormat[T](
new Tuple8(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7))
.asInstanceOf[T]
case 9 =>
- new Tuple9(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8)).asInstanceOf[T]
+ new Tuple9(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7), vals(8))
+ .asInstanceOf[T]
case 10 =>
- new Tuple10(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9)).asInstanceOf[T]
+ new Tuple10(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9)).asInstanceOf[T]
case 11 =>
- new Tuple11(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10)).asInstanceOf[T]
+ new Tuple11(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10)).asInstanceOf[T]
case 12 =>
- new Tuple12(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11)).asInstanceOf[T]
+ new Tuple12(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11)).asInstanceOf[T]
case 13 =>
- new Tuple13(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12)).asInstanceOf[T]
+ new Tuple13(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12)).asInstanceOf[T]
case 14 =>
- new Tuple14(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13)).asInstanceOf[T]
+ new Tuple14(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13)).asInstanceOf[T]
case 15 =>
- new Tuple15(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14)).asInstanceOf[T]
+ new Tuple15(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14)).asInstanceOf[T]
case 16 =>
- new Tuple16(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15))
+ new Tuple16(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15))
.asInstanceOf[T]
case 17 =>
- new Tuple17(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
+ new Tuple17(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
vals(16)).asInstanceOf[T]
case 18 =>
- new Tuple18(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17)).asInstanceOf[T]
+ new Tuple18(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17)).asInstanceOf[T]
case 19 =>
- new Tuple19(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18)).asInstanceOf[T]
+ new Tuple19(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18)).asInstanceOf[T]
case 20 =>
- new Tuple20(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18), vals(19)).asInstanceOf[T]
+ new Tuple20(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18),
+ vals(19)).asInstanceOf[T]
case 21 =>
- new Tuple21(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18), vals(19), vals(20)).asInstanceOf[T]
+ new Tuple21(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18),
+ vals(19),
+ vals(20)).asInstanceOf[T]
case 22 =>
- new Tuple22(vals(0), vals(1), vals(2), vals(3), vals(4), vals(5), vals(6), vals(7),
- vals(8), vals(9), vals(10), vals(11), vals(12), vals(13), vals(14), vals(15),
- vals(16), vals(17), vals(18), vals(19), vals(20), vals(21)).asInstanceOf[T]
+ new Tuple22(
+ vals(0),
+ vals(1),
+ vals(2),
+ vals(3),
+ vals(4),
+ vals(5),
+ vals(6),
+ vals(7),
+ vals(8),
+ vals(9),
+ vals(10),
+ vals(11),
+ vals(12),
+ vals(13),
+ vals(14),
+ vals(15),
+ vals(16),
+ vals(17),
+ vals(18),
+ vals(19),
+ vals(20),
+ vals(21)).asInstanceOf[T]
case _ =>
throw new RuntimeException("Only up to 22 fields supported for Scala Tuples.")
- }
+ }
}
}
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala b/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala
index 8af5eec65b7..46e7d8098a0 100644
--- a/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-scala/src/main/scala/org/apache/flink/tests/scala/ScalaJob.scala
@@ -34,13 +34,15 @@ object ScalaJob {
// we want to go through serialization to check for kryo issues
env.disableOperatorChaining()
- env.fromElements(new NonPojo()).map(new MapFunction[NonPojo, NonPojo] {
- override def map(value: NonPojo): NonPojo = {
- // use some method that was removed in 2.12+
- BoxesRunTime.hashFromNumber(value.getSomeInt)
- value
- }
- })
+ env
+ .fromElements(new NonPojo())
+ .map(new MapFunction[NonPojo, NonPojo] {
+ override def map(value: NonPojo): NonPojo = {
+ // use some method that was removed in 2.12+
+ BoxesRunTime.hashFromNumber(value.getSomeInt)
+ value
+ }
+ })
env.execute();
}
diff --git a/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala b/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala
index 74c7a13b16f..985b2a62296 100644
--- a/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala
+++ b/flink-end-to-end-tests/flink-quickstart-test/src/main/scala/org/apache/flink/quickstarts/test/Elasticsearch7SinkExample.scala
@@ -1,12 +1,13 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.quickstarts.test
import org.apache.flink.api.connector.sink2.SinkWriter
@@ -22,28 +22,30 @@ import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.connector.elasticsearch.sink.{Elasticsearch7SinkBuilder, RequestIndexer}
import org.apache.flink.streaming.api.datastream.DataStream
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+
import org.apache.http.HttpHost
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.client.Requests
import scala.collection.JavaConversions.mapAsJavaMap
-
object Elasticsearch7SinkExample {
def main(args: Array[String]) {
val parameterTool = ParameterTool.fromArgs(args)
if (parameterTool.getNumberOfParameters < 3) {
- println("Missing parameters!\n" + "Usage:" +
- " --numRecords <numRecords> --index <index> --type <type>")
+ println(
+ "Missing parameters!\n" + "Usage:" +
+ " --numRecords <numRecords> --index <index> --type <type>")
return
}
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.enableCheckpointing(5000)
- val source: DataStream[(String)] = env.fromSequence(0, 20 - 1)
- .map(v => "message #" + v.toString)
+ val source: DataStream[(String)] = env
+ .fromSequence(0, 20 - 1)
+ .map(v => "message #" + v.toString)
source.sinkTo(
new Elasticsearch7SinkBuilder[String]
@@ -51,8 +53,9 @@ object Elasticsearch7SinkExample {
// be buffered
.setBulkFlushMaxActions(1)
.setHosts(new HttpHost("127.0.0.1", 9200, "http"))
- .setEmitter((element: String, context: SinkWriter.Context, indexer: RequestIndexer) =>
- indexer.add(createIndexRequest(element, parameterTool)))
+ .setEmitter(
+ (element: String, context: SinkWriter.Context, indexer: RequestIndexer) =>
+ indexer.add(createIndexRequest(element, parameterTool)))
.build())
env.execute("Elasticsearch7.x end to end sink test example")
@@ -64,7 +67,9 @@ object Elasticsearch7SinkExample {
"data" -> element.asInstanceOf[AnyRef]
)
- Requests.indexRequest.index(parameterTool.getRequired("index"))
- .`type`(parameterTool.getRequired("type")).source(mapAsJavaMap(json2))
+ Requests.indexRequest
+ .index(parameterTool.getRequired("index"))
+ .`type`(parameterTool.getRequired("type"))
+ .source(mapAsJavaMap(json2))
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala
index e5a16a51783..2d4623bc473 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/clustering/KMeans.scala
@@ -29,46 +29,40 @@ import scala.collection.JavaConverters._
/**
* This example implements a basic K-Means clustering algorithm.
*
- * K-Means is an iterative clustering algorithm and works as follows:
- * K-Means is given a set of data points to be clustered and an initial set of ''K'' cluster
- * centers.
- * In each iteration, the algorithm computes the distance of each data point to each cluster center.
- * Each point is assigned to the cluster center which is closest to it.
- * Subsequently, each cluster center is moved to the center (''mean'') of all points that have
- * been assigned to it.
- * The moved cluster centers are fed into the next iteration.
- * The algorithm terminates after a fixed number of iterations (as in this implementation)
- * or if cluster centers do not (significantly) move in an iteration.
- * This is the Wikipedia entry for the [[http://en.wikipedia
- * .org/wiki/K-means_clustering K-Means Clustering algorithm]].
+ * K-Means is an iterative clustering algorithm and works as follows: K-Means is given a set of data
+ * points to be clustered and an initial set of ''K'' cluster centers. In each iteration, the
+ * algorithm computes the distance of each data point to each cluster center. Each point is assigned
+ * to the cluster center which is closest to it. Subsequently, each cluster center is moved to the
+ * center (''mean'') of all points that have been assigned to it. The moved cluster centers are fed
+ * into the next iteration. The algorithm terminates after a fixed number of iterations (as in this
+ * implementation) or if cluster centers do not (significantly) move in an iteration. This is the
+ * Wikipedia entry for the
+ * [[http://en.wikipedia .org/wiki/K-means_clustering K-Means Clustering algorithm]].
*
- * This implementation works on two-dimensional data points.
- * It computes an assignment of data points to cluster centers, i.e.,
- * each data point is annotated with the id of the final cluster (center) it belongs to.
+ * This implementation works on two-dimensional data points. It computes an assignment of data
+ * points to cluster centers, i.e., each data point is annotated with the id of the final cluster
+ * (center) it belongs to.
*
* Input files are plain text files and must be formatted as follows:
*
- * - Data points are represented as two double values separated by a blank character.
- * Data points are separated by newline characters.
- * For example `"1.2 2.3\n5.3 7.2\n"` gives two data points (x=1.2, y=2.3) and (x=5.3,
- * y=7.2).
- * - Cluster centers are represented by an integer id and a point value.
- * For example `"1 6.2 3.2\n2 2.9 5.7\n"` gives two centers (id=1, x=6.2,
- * y=3.2) and (id=2, x=2.9, y=5.7).
+ * - Data points are represented as two double values separated by a blank character. Data points
+ * are separated by newline characters. For example `"1.2 2.3\n5.3 7.2\n"` gives two data points
+ * (x=1.2, y=2.3) and (x=5.3, y=7.2).
+ * - Cluster centers are represented by an integer id and a point value. For example `"1 6.2
+ * 3.2\n2 2.9 5.7\n"` gives two centers (id=1, x=6.2, y=3.2) and (id=2, x=2.9, y=5.7).
*
* Usage:
* {{{
* KMeans --points <path> --centroids <path> --output <path> --iterations <n>
* }}}
* If no parameters are provided, the program is run with default data from
- * [[org.apache.flink.examples.java.clustering.util.KMeansData]]
- * and 10 iterations.
+ * [[org.apache.flink.examples.java.clustering.util.KMeansData]] and 10 iterations.
*
* This example shows how to use:
*
- * - Bulk iterations
- * - Broadcast variables in bulk iterations
- * - Scala case classes
+ * - Bulk iterations
+ * - Broadcast variables in bulk iterations
+ * - Scala case classes
*/
object KMeans {
@@ -85,14 +79,19 @@ object KMeans {
val points: DataSet[Point] = getPointDataSet(params, env)
val centroids: DataSet[Centroid] = getCentroidDataSet(params, env)
- val finalCentroids = centroids.iterate(params.getInt("iterations", 10)) { currentCentroids =>
- val newCentroids = points
- .map(new SelectNearestCenter).withBroadcastSet(currentCentroids, "centroids")
- .map { x => (x._1, x._2, 1L) }.withForwardedFields("_1; _2")
- .groupBy(0)
- .reduce { (p1, p2) => (p1._1, p1._2.add(p2._2), p1._3 + p2._3) }.withForwardedFields("_1")
- .map { x => new Centroid(x._1, x._2.div(x._3)) }.withForwardedFields("_1->id")
- newCentroids
+ val finalCentroids = centroids.iterate(params.getInt("iterations", 10)) {
+ currentCentroids =>
+ val newCentroids = points
+ .map(new SelectNearestCenter)
+ .withBroadcastSet(currentCentroids, "centroids")
+ .map(x => (x._1, x._2, 1L))
+ .withForwardedFields("_1; _2")
+ .groupBy(0)
+ .reduce((p1, p2) => (p1._1, p1._2.add(p2._2), p1._3 + p2._3))
+ .withForwardedFields("_1")
+ .map(x => new Centroid(x._1, x._2.div(x._3)))
+ .withForwardedFields("_1->id")
+ newCentroids
}
val clusteredPoints: DataSet[(Int, Point)] =
@@ -121,7 +120,7 @@ object KMeans {
} else {
println("Executing K-Means example with default centroid data set.")
println("Use --centroids to specify file input.")
- env.fromCollection(KMeansData.CENTROIDS map {
+ env.fromCollection(KMeansData.CENTROIDS.map {
case Array(id, x, y) =>
new Centroid(id.asInstanceOf[Int], x.asInstanceOf[Double], y.asInstanceOf[Double])
})
@@ -137,7 +136,7 @@ object KMeans {
} else {
println("Executing K-Means example with default points data set.")
println("Use --points to specify file input.")
- env.fromCollection(KMeansData.POINTS map {
+ env.fromCollection(KMeansData.POINTS.map {
case Array(x, y) => new Point(x.asInstanceOf[Double], y.asInstanceOf[Double])
})
}
@@ -148,9 +147,9 @@ object KMeans {
// *************************************************************************
/**
- * Common trait for operations supported by both points and centroids
- * Note: case class inheritance is not allowed in Scala
- */
+ * Common trait for operations supported by both points and centroids Note: case class inheritance
+ * is not allowed in Scala
+ */
trait Coordinate extends Serializable {
var x: Double
@@ -181,14 +180,10 @@ object KMeans {
}
- /**
- * A simple two-dimensional point.
- */
+ /** A simple two-dimensional point. */
case class Point(var x: Double = 0, var y: Double = 0) extends Coordinate
- /**
- * A simple two-dimensional centroid, basically a point with an ID.
- */
+ /** A simple two-dimensional centroid, basically a point with an ID. */
case class Centroid(var id: Int = 0, var x: Double = 0, var y: Double = 0) extends Coordinate {
def this(id: Int, p: Point) {
@@ -225,5 +220,3 @@ object KMeans {
}
}
-
-
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
index c7186123c0e..8f58c7af8d3 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.java.utils.ParameterTool
@@ -27,39 +26,36 @@ import org.apache.flink.util.Collector
* An implementation of the connected components algorithm, using a delta iteration.
*
* Initially, the algorithm assigns each vertex an unique ID. In each step, a vertex picks the
- * minimum of its own ID and its neighbors' IDs, as its new ID and tells its neighbors about its
- * new ID. After the algorithm has completed, all vertices in the same component will have the same
- * ID.
+ * minimum of its own ID and its neighbors' IDs, as its new ID and tells its neighbors about its new
+ * ID. After the algorithm has completed, all vertices in the same component will have the same ID.
*
- * A vertex whose component ID did not change needs not propagate its information in the next
- * step. Because of that, the algorithm is easily expressible via a delta iteration. We here model
- * the solution set as the vertices with their current component ids, and the workset as the changed
+ * A vertex whose component ID did not change needs not propagate its information in the next step.
+ * Because of that, the algorithm is easily expressible via a delta iteration. We here model the
+ * solution set as the vertices with their current component ids, and the workset as the changed
* vertices. Because we see all vertices initially as changed, the initial workset and the initial
* solution set are identical. Also, the delta to the solution set is consequently also the next
* workset.
- *
+ *
* Input files are plain text files and must be formatted as follows:
*
* - Vertices represented as IDs and separated by new-line characters. For example,
* `"1\n2\n12\n42\n63"` gives five vertices (1), (2), (12), (42), and (63).
* - Edges are represented as pairs for vertex IDs which are separated by space characters. Edges
- * are separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"`
- * gives four (undirected) edges (1)-(2), (2)-(12), (1)-(12), and (42)-(63).
+ * are separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"` gives four
+ * (undirected) edges (1)-(2), (2)-(12), (1)-(12), and (42)-(63).
*
* Usage:
* {{{
* ConnectedComponents --vertices <path> --edges <path> --result <path> --iterations <n>
* }}}
- *
+ *
* If no parameters are provided, the program is run with default data from
* [[org.apache.flink.examples.java.graph.util.ConnectedComponentsData]] and 10 iterations.
- *
*
* This example shows how to use:
*
* - Delta Iterations
- * - Generic-typed Functions
- *
+ * - Generic-typed Functions
*/
object ConnectedComponents {
@@ -78,30 +74,36 @@ object ConnectedComponents {
// read vertex and edge data
// assign the initial components (equal to the vertex id)
val vertices =
- getVertexDataSet(env, params).map { id => (id, id) }.withForwardedFields("*->_1;*->_2")
+ getVertexDataSet(env, params).map(id => (id, id)).withForwardedFields("*->_1;*->_2")
// undirected edges by emitting for each input edge the input
// edges itself and an inverted version
val edges =
- getEdgeDataSet(env, params).flatMap { edge => Seq(edge, (edge._2, edge._1)) }
+ getEdgeDataSet(env, params).flatMap(edge => Seq(edge, (edge._2, edge._1)))
// open a delta iteration
val verticesWithComponents = vertices.iterateDelta(vertices, maxIterations, Array("_1")) {
(s, ws) =>
-
// apply the step logic: join with the edges
- val allNeighbors = ws.join(edges).where(0).equalTo(0) { (vertex, edge) =>
- (edge._2, vertex._2)
- }.withForwardedFieldsFirst("_2->_2").withForwardedFieldsSecond("_2->_1")
+ val allNeighbors = ws
+ .join(edges)
+ .where(0)
+ .equalTo(0)((vertex, edge) => (edge._2, vertex._2))
+ .withForwardedFieldsFirst("_2->_2")
+ .withForwardedFieldsSecond("_2->_1")
// select the minimum neighbor
val minNeighbors = allNeighbors.groupBy(0).min(1)
// update if the component of the candidate is smaller
- val updatedComponents = minNeighbors.join(s).where(0).equalTo(0) {
- (newVertex, oldVertex, out: Collector[(Long, Long)]) =>
- if (newVertex._2 < oldVertex._2) out.collect(newVertex)
- }.withForwardedFieldsFirst("*")
+ val updatedComponents = minNeighbors
+ .join(s)
+ .where(0)
+ .equalTo(0) {
+ (newVertex, oldVertex, out: Collector[(Long, Long)]) =>
+ if (newVertex._2 < oldVertex._2) out.collect(newVertex)
+ }
+ .withForwardedFieldsFirst("*")
// delta and new workset are identical
(updatedComponents, updatedComponents)
@@ -119,31 +121,30 @@ object ConnectedComponents {
private def getVertexDataSet(env: ExecutionEnvironment, params: ParameterTool): DataSet[Long] = {
if (params.has("vertices")) {
- env.readCsvFile[Tuple1[Long]](
- params.get("vertices"),
- includedFields = Array(0))
- .map { x => x._1 }
- }
- else {
+ env
+ .readCsvFile[Tuple1[Long]](params.get("vertices"), includedFields = Array(0))
+ .map(x => x._1)
+ } else {
println("Executing ConnectedComponents example with default vertices data set.")
println("Use --vertices to specify file input.")
env.fromCollection(ConnectedComponentsData.VERTICES)
}
}
- private def getEdgeDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(Long, Long)] = {
+ private def getEdgeDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(Long, Long)] = {
if (params.has("edges")) {
- env.readCsvFile[(Long, Long)](
- params.get("edges"),
- fieldDelimiter = " ",
- includedFields = Array(0, 1))
- .map { x => (x._1, x._2)}
- }
- else {
+ env
+ .readCsvFile[(Long, Long)](
+ params.get("edges"),
+ fieldDelimiter = " ",
+ includedFields = Array(0, 1))
+ .map(x => (x._1, x._2))
+ } else {
println("Executing ConnectedComponents example with default edges data set.")
println("Use --edges to specify file input.")
- val edgeData = ConnectedComponentsData.EDGES map {
+ val edgeData = ConnectedComponentsData.EDGES.map {
case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
}
env.fromCollection(edgeData)
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
index 159891bad4c..2649890a2c4 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.scala._
@@ -23,11 +22,11 @@ import org.apache.flink.util.Collector
object DeltaPageRank {
- private final val DAMPENING_FACTOR: Double = 0.85
- private final val NUM_VERTICES = 5
- private final val INITIAL_RANK = 1.0 / NUM_VERTICES
- private final val RANDOM_JUMP = (1 - DAMPENING_FACTOR) / NUM_VERTICES
- private final val THRESHOLD = 0.0001 / NUM_VERTICES
+ final private val DAMPENING_FACTOR: Double = 0.85
+ final private val NUM_VERTICES = 5
+ final private val INITIAL_RANK = 1.0 / NUM_VERTICES
+ final private val RANDOM_JUMP = (1 - DAMPENING_FACTOR) / NUM_VERTICES
+ final private val THRESHOLD = 0.0001 / NUM_VERTICES
type Page = (Long, Double)
type Adjacency = (Long, Array[Long])
@@ -38,61 +37,66 @@ object DeltaPageRank {
val env = ExecutionEnvironment.getExecutionEnvironment
- val rawLines: DataSet[String] = env.fromElements(
- "1 2 3 4",
- "2 1",
- "3 5",
- "4 2 3",
- "5 2 4")
+ val rawLines: DataSet[String] = env.fromElements("1 2 3 4", "2 1", "3 5", "4 2 3", "5 2 4")
val adjacency: DataSet[Adjacency] = rawLines
- .map(str => {
- val elements = str.split(' ')
- val id = elements(0).toLong
- val neighbors = elements.slice(1, elements.length).map(_.toLong)
- (id, neighbors)
- })
-
- val initialRanks: DataSet[Page] = adjacency.flatMap {
- (adj, out: Collector[Page]) =>
- {
- val targets = adj._2
- val rankPerTarget = INITIAL_RANK * DAMPENING_FACTOR / targets.length
-
- // dampen fraction to targets
- for (target <- targets) {
- out.collect((target, rankPerTarget))
+ .map(
+ str => {
+ val elements = str.split(' ')
+ val id = elements(0).toLong
+ val neighbors = elements.slice(1, elements.length).map(_.toLong)
+ (id, neighbors)
+ })
+
+ val initialRanks: DataSet[Page] = adjacency
+ .flatMap {
+ (adj, out: Collector[Page]) =>
+ {
+ val targets = adj._2
+ val rankPerTarget = INITIAL_RANK * DAMPENING_FACTOR / targets.length
+
+ // dampen fraction to targets
+ for (target <- targets) {
+ out.collect((target, rankPerTarget))
+ }
+
+ // random jump to self
+ out.collect((adj._1, RANDOM_JUMP))
}
+ }
+ .groupBy(0)
+ .sum(1)
- // random jump to self
- out.collect((adj._1, RANDOM_JUMP))
- }
- }
- .groupBy(0).sum(1)
-
- val initialDeltas = initialRanks.map { (page) => (page._1, page._2 - INITIAL_RANK) }
- .withForwardedFields("_1")
+ val initialDeltas = initialRanks
+ .map((page) => (page._1, page._2 - INITIAL_RANK))
+ .withForwardedFields("_1")
val iteration = initialRanks.iterateDelta(initialDeltas, maxIterations, Array(0)) {
(solutionSet, workset) =>
{
- val deltas = workset.join(adjacency).where(0).equalTo(0) {
- (lastDeltas, adj, out: Collector[Page]) =>
- {
- val targets = adj._2
- val deltaPerTarget = DAMPENING_FACTOR * lastDeltas._2 / targets.length
-
- for (target <- targets) {
- out.collect((target, deltaPerTarget))
+ val deltas = workset
+ .join(adjacency)
+ .where(0)
+ .equalTo(0) {
+ (lastDeltas, adj, out: Collector[Page]) =>
+ {
+ val targets = adj._2
+ val deltaPerTarget = DAMPENING_FACTOR * lastDeltas._2 / targets.length
+
+ for (target <- targets) {
+ out.collect((target, deltaPerTarget))
+ }
}
- }
- }
- .groupBy(0).sum(1)
+ }
+ .groupBy(0)
+ .sum(1)
.filter(x => Math.abs(x._2) > THRESHOLD)
- val rankUpdates = solutionSet.join(deltas).where(0).equalTo(0) {
- (current, delta) => (current._1, current._2 + delta._2)
- }.withForwardedFieldsFirst("_1")
+ val rankUpdates = solutionSet
+ .join(deltas)
+ .where(0)
+ .equalTo(0)((current, delta) => (current._1, current._2 + delta._2))
+ .withForwardedFieldsFirst("_1")
(rankUpdates, deltas)
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala
index 32adeb59274..a91ad34087d 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/EnumTriangles.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.common.functions.GroupReduceFunction
@@ -30,40 +29,32 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
/**
- * Triangle enumeration is a pre-processing step to find closely connected parts in graphs.
- * A triangle consists of three edges that connect three vertices with each other.
- *
- * The algorithm works as follows:
- * It groups all edges that share a common vertex and builds triads, i.e., triples of vertices
- * that are connected by two edges. Finally, all triads are filtered for which no third edge exists
- * that closes the triangle.
- *
+ * Triangle enumeration is a pre-processing step to find closely connected parts in graphs. A
+ * triangle consists of three edges that connect three vertices with each other.
+ *
+ * The algorithm works as follows: It groups all edges that share a common vertex and builds triads,
+ * i.e., triples of vertices that are connected by two edges. Finally, all triads are filtered for
+ * which no third edge exists that closes the triangle.
+ *
* Input files are plain text files and must be formatted as follows:
*
- * - Edges are represented as pairs for vertex IDs which are separated by space
- * characters. Edges are separated by new-line characters.
- * For example `"1 2\n2 12\n1 12\n42 63"` gives four (undirected) edges (1)-(2), (2)-(12),
- * (1)-(12), and (42)-(63) that include a triangle
+ * - Edges are represented as pairs for vertex IDs which are separated by space characters. Edges
+ * are separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"` gives four
+ * (undirected) edges (1)-(2), (2)-(12), (1)-(12), and (42)-(63) that include a triangle
+ *
+ * <pre> (1) / \ (2)-(12) </pre>
*
- * <pre>
- * (1)
- * / \
- * (2)-(12)
- * </pre>
- *
- * Usage:
+ * Usage:
* {{{
* EnumTriangleBasic <edge path> <result path>
* }}}
- * <br>
- * If no parameters are provided, the program is run with default data from
+ * <br> If no parameters are provided, the program is run with default data from
* [[org.apache.flink.examples.java.graph.util.EnumTrianglesData]]
- *
- * This example shows how to use:
*
- * - Custom Java objects which extend Tuple
- * - Group Sorting
+ * This example shows how to use:
*
+ * - Custom Java objects which extend Tuple
+ * - Group Sorting
*/
object EnumTriangles {
@@ -92,17 +83,21 @@ object EnumTriangles {
}
env.fromCollection(edges)
}
-
+
// project edges by vertex id
- val edgesById = edges map(e => if (e.v1 < e.v2) e else Edge(e.v2, e.v1) )
-
+ val edgesById = edges.map(e => if (e.v1 < e.v2) e else Edge(e.v2, e.v1))
+
val triangles = edgesById
- // build triads
- .groupBy("v1").sortGroup("v2", Order.ASCENDING).reduceGroup(new TriadBuilder())
- // filter triads
- .join(edgesById).where("v2", "v3").equalTo("v1", "v2") { (t, _) => t }
- .withForwardedFieldsFirst("*")
-
+ // build triads
+ .groupBy("v1")
+ .sortGroup("v2", Order.ASCENDING)
+ .reduceGroup(new TriadBuilder())
+ // filter triads
+ .join(edgesById)
+ .where("v2", "v3")
+ .equalTo("v1", "v2")((t, _) => t)
+ .withForwardedFieldsFirst("*")
+
// emit result
if (params.has("output")) {
triangles.writeAsCsv(params.get("output"), "\n", ",")
@@ -112,7 +107,6 @@ object EnumTriangles {
println("Printing result to stdout. Use --output to specify output path.")
triangles.print()
}
-
}
@@ -122,32 +116,31 @@ object EnumTriangles {
case class Edge(v1: Int, v2: Int) extends Serializable
case class Triad(v1: Int, v2: Int, v3: Int) extends Serializable
-
-
+
// *************************************************************************
// USER FUNCTIONS
// *************************************************************************
/**
- * Builds triads (triples of vertices) from pairs of edges that share a vertex. The first vertex
- * of a triad is the shared vertex, the second and third vertex are ordered by vertexId. Assumes
- * that input edges share the first vertex and are in ascending order of the second vertex.
+ * Builds triads (triples of vertices) from pairs of edges that share a vertex. The first vertex
+ * of a triad is the shared vertex, the second and third vertex are ordered by vertexId. Assumes
+ * that input edges share the first vertex and are in ascending order of the second vertex.
*/
@ForwardedFields(Array("v1->v1"))
class TriadBuilder extends GroupReduceFunction[Edge, Triad] {
val vertices = mutable.MutableList[Integer]()
-
+
override def reduce(edges: java.lang.Iterable[Edge], out: Collector[Triad]) = {
-
+
// clear vertex list
vertices.clear()
// build and emit triads
- for(e <- edges.asScala) {
-
+ for (e <- edges.asScala) {
+
// combine vertex with all previously read vertices
- for(v <- vertices) {
+ for (v <- vertices) {
out.collect(Triad(e.v1, v, e.v2))
}
vertices += e.v2
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
index 4692dbd0fb7..afc1a44f1a1 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
@@ -17,8 +17,6 @@
*/
package org.apache.flink.examples.scala.graph
-import java.lang.Iterable
-
import org.apache.flink.api.common.functions.GroupReduceFunction
import org.apache.flink.api.java.aggregation.Aggregations.SUM
import org.apache.flink.api.java.utils.ParameterTool
@@ -26,11 +24,13 @@ import org.apache.flink.api.scala._
import org.apache.flink.examples.java.graph.util.PageRankData
import org.apache.flink.util.Collector
+import java.lang.Iterable
+
import scala.collection.JavaConverters._
/**
* A basic implementation of the Page Rank algorithm using a bulk iteration.
- *
+ *
* This implementation requires a set of pages and a set of directed links as input and works as
* follows.
*
@@ -40,16 +40,16 @@ import scala.collection.JavaConverters._
* with the new ranks of all pages. This implementation terminates after a fixed number of
* iterations. This is the Wikipedia entry for the
* [[http://en.wikipedia.org/wiki/Page_rank Page Rank algorithm]]
- *
+ *
* Input files are plain text files and must be formatted as follows:
*
- * - Pages represented as an (long) ID separated by new-line characters.
- * For example `"1\n2\n12\n42\n63"` gives five pages with IDs 1, 2, 12, 42, and 63.
- * - Links are represented as pairs of page IDs which are separated by space characters. Links
- * are separated by new-line characters.
- * For example `"1 2\n2 12\n1 12\n42 63"` gives four (directed) links (1)->(2), (2)->(12),
- * (1)->(12), and (42)->(63). For this simple implementation it is required that each page has
- * at least one incoming and one outgoing link (a page can point to itself).
+ * - Pages represented as an (long) ID separated by new-line characters. For example
+ * `"1\n2\n12\n42\n63"` gives five pages with IDs 1, 2, 12, 42, and 63.
+ * - Links are represented as pairs of page IDs which are separated by space characters. Links are
+ * separated by new-line characters. For example `"1 2\n2 12\n1 12\n42 63"` gives four
+ * (directed) links (1)->(2), (2)->(12), (1)->(12), and (42)->(63). For this simple
+ * implementation it is required that each page has at least one incoming and one outgoing link
+ * (a page can point to itself).
*
* Usage:
* {{{
@@ -58,18 +58,17 @@ import scala.collection.JavaConverters._
*
* If no parameters are provided, the program is run with default data from
* [[org.apache.flink.examples.java.graph.util.PageRankData]] and 10 iterations.
- *
- * This example shows how to use:
*
- * - Bulk Iterations
- * - Default Join
- * - Configure user-defined functions using constructor parameters.
+ * This example shows how to use:
*
+ * - Bulk Iterations
+ * - Default Join
+ * - Configure user-defined functions using constructor parameters.
*/
object PageRankBasic {
- private final val DAMPENING_FACTOR: Double = 0.85
- private final val EPSILON: Double = 0.0001
+ final private val DAMPENING_FACTOR: Double = 0.85
+ final private val EPSILON: Double = 0.0001
def main(args: Array[String]) {
@@ -91,10 +90,11 @@ object PageRankBasic {
// build adjacency list from link input
val adjacencyLists = links
- .groupBy("sourceId").reduceGroup( new GroupReduceFunction[Link, AdjacencyList] {
+ .groupBy("sourceId")
+ .reduceGroup(new GroupReduceFunction[Link, AdjacencyList] {
override def reduce(values: Iterable[Link], out: Collector[AdjacencyList]): Unit = {
var outputId = -1L
- val outputList = values.asScala map { t => outputId = t.sourceId; t.targetId }
+ val outputList = values.asScala.map { t => outputId = t.sourceId; t.targetId }
out.collect(new AdjacencyList(outputId, outputList.toArray))
}
})
@@ -104,18 +104,22 @@ object PageRankBasic {
currentRanks =>
val newRanks = currentRanks
// distribute ranks to target pages
- .join(adjacencyLists).where("pageId").equalTo("sourceId") {
+ .join(adjacencyLists)
+ .where("pageId")
+ .equalTo("sourceId") {
(page, adjacent, out: Collector[Page]) =>
val targets = adjacent.targetIds
val len = targets.length
- adjacent.targetIds foreach { t => out.collect(Page(t, page.rank /len )) }
+ adjacent.targetIds.foreach(t => out.collect(Page(t, page.rank / len)))
}
// collect ranks and sum them up
- .groupBy("pageId").aggregate(SUM, "rank")
+ .groupBy("pageId")
+ .aggregate(SUM, "rank")
// apply dampening factor
- .map { p =>
- Page(p.pageId, (p.rank * DAMPENING_FACTOR) + ((1 - DAMPENING_FACTOR) / numPages))
- }.withForwardedFields("pageId")
+ .map {
+ p => Page(p.pageId, (p.rank * DAMPENING_FACTOR) + ((1 - DAMPENING_FACTOR) / numPages))
+ }
+ .withForwardedFields("pageId")
// terminate if no rank update was significant
val termination = currentRanks.join(newRanks).where("pageId").equalTo("pageId") {
@@ -153,8 +157,9 @@ object PageRankBasic {
// UTIL METHODS
// *************************************************************************
- private def getPagesDataSet(env: ExecutionEnvironment, params: ParameterTool):
- (DataSet[Long], Long) = {
+ private def getPagesDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): (DataSet[Long], Long) = {
if (params.has("pages") && params.has("numPages")) {
val pages = env
.readCsvFile[Tuple1[Long]](params.get("pages"), fieldDelimiter = " ", lineDelimiter = "\n")
@@ -167,16 +172,15 @@ object PageRankBasic {
}
}
- private def getLinksDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[Link] = {
+ private def getLinksDataSet(env: ExecutionEnvironment, params: ParameterTool): DataSet[Link] = {
if (params.has("links")) {
- env.readCsvFile[Link](params.get("links"), fieldDelimiter = " ",
- includedFields = Array(0, 1))
+ env.readCsvFile[Link](params.get("links"), fieldDelimiter = " ", includedFields = Array(0, 1))
} else {
println("Executing PageRank example with default links data set.")
println("Use --links to specify file input.")
- val edges = PageRankData.EDGES.map { case Array(v1, v2) => Link(v1.asInstanceOf[Long],
- v2.asInstanceOf[Long])}
+ val edges = PageRankData.EDGES.map {
+ case Array(v1, v2) => Link(v1.asInstanceOf[Long], v2.asInstanceOf[Long])
+ }
env.fromCollection(edges)
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
index 8f87b91bbed..c20abf15f86 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.graph
import org.apache.flink.api.java.utils.ParameterTool
@@ -25,7 +24,7 @@ import org.apache.flink.util.Collector
object TransitiveClosureNaive {
- def main (args: Array[String]): Unit = {
+ def main(args: Array[String]): Unit = {
val params: ParameterTool = ParameterTool.fromArgs(args)
@@ -37,15 +36,16 @@ object TransitiveClosureNaive {
val edges =
if (params.has("edges")) {
- env.readCsvFile[(Long, Long)](
- filePath = params.get("edges"),
- fieldDelimiter = " ",
- includedFields = Array(0, 1))
- .map { x => (x._1, x._2)}
+ env
+ .readCsvFile[(Long, Long)](
+ filePath = params.get("edges"),
+ fieldDelimiter = " ",
+ includedFields = Array(0, 1))
+ .map(x => (x._1, x._2))
} else {
println("Executing TransitiveClosure example with default edges data set.")
println("Use --edges to specify file input.")
- val edgeData = ConnectedComponentsData.EDGES map {
+ val edgeData = ConnectedComponentsData.EDGES.map {
case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
}
env.fromCollection(edgeData)
@@ -53,30 +53,35 @@ object TransitiveClosureNaive {
val maxIterations = params.getInt("iterations", 10)
- val paths = edges.iterateWithTermination(maxIterations) { prevPaths: DataSet[(Long, Long)] =>
-
- val nextPaths = prevPaths
- .join(edges)
- .where(1).equalTo(0) {
- (left, right) => (left._1,right._2)
- }.withForwardedFieldsFirst("_1").withForwardedFieldsSecond("_2")
- .union(prevPaths)
- .groupBy(0, 1)
- .reduce((l, r) => l).withForwardedFields("_1; _2")
+ val paths = edges.iterateWithTermination(maxIterations) {
+ prevPaths: DataSet[(Long, Long)] =>
+ val nextPaths = prevPaths
+ .join(edges)
+ .where(1)
+ .equalTo(0)((left, right) => (left._1, right._2))
+ .withForwardedFieldsFirst("_1")
+ .withForwardedFieldsSecond("_2")
+ .union(prevPaths)
+ .groupBy(0, 1)
+ .reduce((l, r) => l)
+ .withForwardedFields("_1; _2")
- val terminate = prevPaths
- .coGroup(nextPaths)
- .where(0).equalTo(0) {
- (
- prev: Iterator[(Long, Long)],
- next: Iterator[(Long, Long)],
- out: Collector[(Long, Long)]) => {
- val prevPaths = prev.toSet
- for (n <- next)
- if (!prevPaths.contains(n)) out.collect(n)
- }
- }.withForwardedFieldsSecond("*")
- (nextPaths, terminate)
+ val terminate = prevPaths
+ .coGroup(nextPaths)
+ .where(0)
+ .equalTo(0) {
+ (
+ prev: Iterator[(Long, Long)],
+ next: Iterator[(Long, Long)],
+ out: Collector[(Long, Long)]) =>
+ {
+ val prevPaths = prev.toSet
+ for (n <- next)
+ if (!prevPaths.contains(n)) out.collect(n)
+ }
+ }
+ .withForwardedFieldsSecond("*")
+ (nextPaths, terminate)
}
if (params.has("output")) {
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala
index 3453ee8bf8b..ee5396f21bb 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/misc/PiEstimation.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.misc
import org.apache.flink.api.scala._
@@ -31,18 +30,20 @@ object PiEstimation {
// count how many of the samples would randomly fall into
// the upper right quadrant of the unit circle
val count =
- env.generateSequence(1, numSamples)
- .map { sample =>
- val x = Math.random()
- val y = Math.random()
- if (x * x + y * y < 1) 1L else 0L
+ env
+ .generateSequence(1, numSamples)
+ .map {
+ sample =>
+ val x = Math.random()
+ val y = Math.random()
+ if (x * x + y * y < 1) 1L else 0L
}
.reduce(_ + _)
// ratio of samples in upper right quadrant vs total samples gives surface of upper
// right quadrant, times 4 gives surface of whole unit circle, i.e. PI
val pi = count
- .map ( _ * 4.0 / numSamples)
+ .map(_ * 4.0 / numSamples)
println("We estimate Pi to be:")
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala
index d510a3630ac..c38e1876b7f 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery10.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.relational
import org.apache.flink.api.java.aggregation.Aggregations
@@ -23,72 +22,74 @@ import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.api.scala._
/**
- * This program implements a modified version of the TPC-H query 10.
- *
+ * This program implements a modified version of the TPC-H query 10.
+ *
* The original query can be found at
- * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf)
- * (page 45).
+ * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf) (page
+ * 45).
*
* This program implements the following SQL equivalent:
*
* {{{
- * SELECT
+ * SELECT
* c_custkey,
- * c_name,
+ * c_name,
* c_address,
- * n_name,
+ * n_name,
* c_acctbal
- * SUM(l_extendedprice * (1 - l_discount)) AS revenue,
- * FROM
- * customer,
- * orders,
- * lineitem,
- * nation
- * WHERE
- * c_custkey = o_custkey
- * AND l_orderkey = o_orderkey
- * AND YEAR(o_orderdate) > '1990'
- * AND l_returnflag = 'R'
- * AND c_nationkey = n_nationkey
- * GROUP BY
- * c_custkey,
- * c_name,
- * c_acctbal,
- * n_name,
+ * SUM(l_extendedprice * (1 - l_discount)) AS revenue,
+ * FROM
+ * customer,
+ * orders,
+ * lineitem,
+ * nation
+ * WHERE
+ * c_custkey = o_custkey
+ * AND l_orderkey = o_orderkey
+ * AND YEAR(o_orderdate) > '1990'
+ * AND l_returnflag = 'R'
+ * AND c_nationkey = n_nationkey
+ * GROUP BY
+ * c_custkey,
+ * c_name,
+ * c_acctbal,
+ * n_name,
* c_address
* }}}
*
- * Compared to the original TPC-H query this version does not print
- * c_phone and c_comment, only filters by years greater than 1990 instead of
- * a period of 3 months, and does not sort the result by revenue..
+ * Compared to the original TPC-H query this version does not print c_phone and c_comment, only
+ * filters by years greater than 1990 instead of a period of 3 months, and does not sort the result
+ * by revenue..
*
- * Input files are plain text CSV files using the pipe character ('|') as field separator
- * as generated by the TPC-H data generator which is available at
- * [http://www.tpc.org/tpch/](a href="http://www.tpc.org/tpch/).
+ * Input files are plain text CSV files using the pipe character ('|') as field separator as
+ * generated by the TPC-H data generator which is available at [http://www.tpc.org/tpch/](a
+ * href="http://www.tpc.org/tpch/).
*
- * Usage:
+ * Usage:
* {{{
- *TPCHQuery10 --customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>
+ * TPCHQuery10 --customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>
* }}}
- *
+ *
* This example shows how to use:
- * - tuple data types
- * - build-in aggregation functions
- * - join with size hints
- *
+ * - tuple data types
+ * - build-in aggregation functions
+ * - join with size hints
*/
object TPCHQuery10 {
def main(args: Array[String]) {
val params: ParameterTool = ParameterTool.fromArgs(args)
- if (!params.has("lineitem") && !params.has("customer") &&
- !params.has("orders") && !params.has("nation")) {
+ if (
+ !params.has("lineitem") && !params.has("customer") &&
+ !params.has("orders") && !params.has("nation")
+ ) {
println(" This program expects data from the TPC-H benchmark as input data.")
println(" Due to legal restrictions, we can not ship generated data.")
println(" You can find the TPC-H data generator at http://www.tpc.org/tpch/.")
- println(" Usage: TPCHQuery10" +
- "--customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>")
+ println(
+ " Usage: TPCHQuery10" +
+ "--customer <path> --orders <path> --lineitem <path> --nation <path> --output <path>")
return
}
@@ -98,34 +99,44 @@ object TPCHQuery10 {
// make parameters available in the web interface
env.getConfig.setGlobalJobParameters(params)
- // get customer data set: (custkey, name, address, nationkey, acctbal)
+ // get customer data set: (custkey, name, address, nationkey, acctbal)
val customers = getCustomerDataSet(env, params.get("customer"))
// get orders data set: (orderkey, custkey, orderdate)
val orders = getOrdersDataSet(env, params.get("orders"))
// get lineitem data set: (orderkey, extendedprice, discount, returnflag)
val lineitems = getLineitemDataSet(env, params.get("lineitem"))
- // get nation data set: (nationkey, name)
+ // get nation data set: (nationkey, name)
val nations = getNationDataSet(env, params.get("nation"))
// filter orders by years
- val orders1990 = orders.filter( o => o._3.substring(0,4).toInt > 1990)
- .map( o => (o._1, o._2))
-
+ val orders1990 = orders
+ .filter(o => o._3.substring(0, 4).toInt > 1990)
+ .map(o => (o._1, o._2))
+
// filter lineitems by return status
- val lineitemsReturn = lineitems.filter( l => l._4.equals("R"))
- .map( l => (l._1, l._2 * (1 - l._3)) )
+ val lineitemsReturn = lineitems
+ .filter(l => l._4.equals("R"))
+ .map(l => (l._1, l._2 * (1 - l._3)))
// compute revenue by customer
- val revenueByCustomer = orders1990.joinWithHuge(lineitemsReturn).where(0).equalTo(0)
- .apply( (o,l) => (o._2, l._2) )
- .groupBy(0)
- .aggregate(Aggregations.SUM, 1)
+ val revenueByCustomer = orders1990
+ .joinWithHuge(lineitemsReturn)
+ .where(0)
+ .equalTo(0)
+ .apply((o, l) => (o._2, l._2))
+ .groupBy(0)
+ .aggregate(Aggregations.SUM, 1)
// compute final result by joining customer and nation information with revenue
- val result = customers.joinWithTiny(nations).where(3).equalTo(0)
- .apply( (c, n) => (c._1, c._2, c._3, n._2, c._5) )
- .join(revenueByCustomer).where(0).equalTo(0)
- .apply( (c, r) => (c._1, c._2, c._3, c._4, c._5, r._2) )
+ val result = customers
+ .joinWithTiny(nations)
+ .where(3)
+ .equalTo(0)
+ .apply((c, n) => (c._1, c._2, c._3, n._2, c._5))
+ .join(revenueByCustomer)
+ .where(0)
+ .equalTo(0)
+ .apply((c, r) => (c._1, c._2, c._3, c._4, c._5, r._2))
if (params.has("output")) {
// emit result
@@ -138,40 +149,41 @@ object TPCHQuery10 {
}
}
-
+
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
- private def getCustomerDataSet(env: ExecutionEnvironment, customerPath: String):
- DataSet[(Int, String, String, Int, Double)] = {
+
+ private def getCustomerDataSet(
+ env: ExecutionEnvironment,
+ customerPath: String): DataSet[(Int, String, String, Int, Double)] = {
env.readCsvFile[(Int, String, String, Int, Double)](
- customerPath,
- fieldDelimiter = "|",
- includedFields = Array(0,1,2,3,5) )
+ customerPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 1, 2, 3, 5))
}
-
- private def getOrdersDataSet(env: ExecutionEnvironment, ordersPath: String):
- DataSet[(Int, Int, String)] = {
+
+ private def getOrdersDataSet(
+ env: ExecutionEnvironment,
+ ordersPath: String): DataSet[(Int, Int, String)] = {
env.readCsvFile[(Int, Int, String)](
- ordersPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 1, 4) )
+ ordersPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 1, 4))
}
-
- private def getLineitemDataSet(env: ExecutionEnvironment, lineitemPath: String):
- DataSet[(Int, Double, Double, String)] = {
+
+ private def getLineitemDataSet(
+ env: ExecutionEnvironment,
+ lineitemPath: String): DataSet[(Int, Double, Double, String)] = {
env.readCsvFile[(Int, Double, Double, String)](
- lineitemPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 5, 6, 8) )
+ lineitemPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 5, 6, 8))
}
- private def getNationDataSet(env: ExecutionEnvironment, nationPath: String):
- DataSet[(Int, String)] = {
- env.readCsvFile[(Int, String)](
- nationPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 1) )
+ private def getNationDataSet(
+ env: ExecutionEnvironment,
+ nationPath: String): DataSet[(Int, String)] = {
+ env.readCsvFile[(Int, String)](nationPath, fieldDelimiter = "|", includedFields = Array(0, 1))
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala
index 2db1fe7f7bb..883ce048118 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.relational
import org.apache.flink.api.java.aggregation.Aggregations
@@ -23,51 +22,49 @@ import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.api.scala._
/**
- * This program implements a modified version of the TPC-H query 3. The
- * example demonstrates how to assign names to fields by extending the Tuple class.
- * The original query can be found at
- * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf)
- * (page 29).
+ * This program implements a modified version of the TPC-H query 3. The example demonstrates how to
+ * assign names to fields by extending the Tuple class. The original query can be found at
+ * [http://www.tpc.org/tpch/spec/tpch2.16.0.pdf](http://www.tpc.org/tpch/spec/tpch2.16.0.pdf) (page
+ * 29).
*
* This program implements the following SQL equivalent:
*
* {{{
- * SELECT
- * l_orderkey,
+ * SELECT
+ * l_orderkey,
* SUM(l_extendedprice*(1-l_discount)) AS revenue,
- * o_orderdate,
- * o_shippriority
- * FROM customer,
- * orders,
- * lineitem
+ * o_orderdate,
+ * o_shippriority
+ * FROM customer,
+ * orders,
+ * lineitem
* WHERE
- * c_mktsegment = '[SEGMENT]'
+ * c_mktsegment = '[SEGMENT]'
* AND c_custkey = o_custkey
* AND l_orderkey = o_orderkey
* AND o_orderdate < date '[DATE]'
* AND l_shipdate > date '[DATE]'
* GROUP BY
- * l_orderkey,
- * o_orderdate,
+ * l_orderkey,
+ * o_orderdate,
* o_shippriority;
* }}}
*
- * Compared to the original TPC-H query this version does not sort the result by revenue
- * and orderdate.
+ * Compared to the original TPC-H query this version does not sort the result by revenue and
+ * orderdate.
*
- * Input files are plain text CSV files using the pipe character ('|') as field separator
- * as generated by the TPC-H data generator which is available at
- * [http://www.tpc.org/tpch/](a href="http://www.tpc.org/tpch/).
+ * Input files are plain text CSV files using the pipe character ('|') as field separator as
+ * generated by the TPC-H data generator which is available at [http://www.tpc.org/tpch/](a
+ * href="http://www.tpc.org/tpch/).
*
- * Usage:
+ * Usage:
* {{{
* TPCHQuery3 --lineitem <path> --customer <path> --orders <path> --output <path>
* }}}
- *
+ *
* This example shows how to use:
- * - case classes and case class field addressing
- * - build-in aggregation functions
- *
+ * - case classes and case class field addressing
+ * - build-in aggregation functions
*/
object TPCHQuery3 {
@@ -78,8 +75,9 @@ object TPCHQuery3 {
println(" This program expects data from the TPC-H benchmark as input data.")
println(" Due to legal restrictions, we can not ship generated data.")
println(" You can find the TPC-H data generator at http://www.tpc.org/tpch/.")
- println(" Usage: TPCHQuery3 " +
- "--lineitem <path> --customer <path> --orders <path> [--output <path>]")
+ println(
+ " Usage: TPCHQuery3 " +
+ "--lineitem <path> --customer <path> --orders <path> [--output <path>]")
return
}
@@ -92,33 +90,37 @@ object TPCHQuery3 {
// set filter date
val dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd")
val date = dateFormat.parse("1995-03-12")
-
+
// read and filter lineitems by shipDate
val lineitems =
- getLineitemDataSet(env, params.get("lineitem")).
- filter( l => dateFormat.parse(l.shipDate).after(date) )
+ getLineitemDataSet(env, params.get("lineitem")).filter(
+ l => dateFormat.parse(l.shipDate).after(date))
// read and filter customers by market segment
val customers =
- getCustomerDataSet(env, params.get("customer")).
- filter( c => c.mktSegment.equals("AUTOMOBILE"))
+ getCustomerDataSet(env, params.get("customer")).filter(c => c.mktSegment.equals("AUTOMOBILE"))
// read orders
val orders = getOrdersDataSet(env, params.get("orders"))
- // filter orders by order date
- val items = orders.filter( o => dateFormat.parse(o.orderDate).before(date) )
- // filter orders by joining with customers
- .join(customers).where("custId").equalTo("custId").apply( (o,c) => o )
- // join with lineitems
- .join(lineitems).where("orderId").equalTo("orderId")
- .apply( (o,l) =>
- new ShippedItem( o.orderId,
- l.extdPrice * (1.0 - l.discount),
- o.orderDate,
- o.shipPrio ) )
+ // filter orders by order date
+ val items = orders
+ .filter(o => dateFormat.parse(o.orderDate).before(date))
+ // filter orders by joining with customers
+ .join(customers)
+ .where("custId")
+ .equalTo("custId")
+ .apply((o, c) => o)
+ // join with lineitems
+ .join(lineitems)
+ .where("orderId")
+ .equalTo("orderId")
+ .apply(
+ (o, l) =>
+ new ShippedItem(o.orderId, l.extdPrice * (1.0 - l.discount), o.orderDate, o.shipPrio))
// group by order and aggregate revenue
- val result = items.groupBy("orderId", "orderDate", "shipPrio")
- .aggregate(Aggregations.SUM, "revenue")
+ val result = items
+ .groupBy("orderId", "orderDate", "shipPrio")
+ .aggregate(Aggregations.SUM, "revenue")
if (params.has("output")) {
// emit result
@@ -131,11 +133,11 @@ object TPCHQuery3 {
}
}
-
+
// *************************************************************************
// USER DATA TYPES
// *************************************************************************
-
+
case class Lineitem(orderId: Long, extdPrice: Double, discount: Double, shipDate: String)
case class Order(orderId: Long, custId: Long, orderDate: String, shipPrio: Long)
case class Customer(custId: Long, mktSegment: String)
@@ -144,28 +146,23 @@ object TPCHQuery3 {
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
- private def getLineitemDataSet(env: ExecutionEnvironment, lineitemPath: String):
- DataSet[Lineitem] = {
+
+ private def getLineitemDataSet(
+ env: ExecutionEnvironment,
+ lineitemPath: String): DataSet[Lineitem] = {
env.readCsvFile[Lineitem](
- lineitemPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 5, 6, 10) )
+ lineitemPath,
+ fieldDelimiter = "|",
+ includedFields = Array(0, 5, 6, 10))
}
- private def getCustomerDataSet(env: ExecutionEnvironment, customerPath: String):
- DataSet[Customer] = {
- env.readCsvFile[Customer](
- customerPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 6) )
+ private def getCustomerDataSet(
+ env: ExecutionEnvironment,
+ customerPath: String): DataSet[Customer] = {
+ env.readCsvFile[Customer](customerPath, fieldDelimiter = "|", includedFields = Array(0, 6))
}
- private def getOrdersDataSet(env: ExecutionEnvironment, ordersPath: String):
- DataSet[Order] = {
- env.readCsvFile[Order](
- ordersPath,
- fieldDelimiter = "|",
- includedFields = Array(0, 1, 4, 7) )
+ private def getOrdersDataSet(env: ExecutionEnvironment, ordersPath: String): DataSet[Order] = {
+ env.readCsvFile[Order](ordersPath, fieldDelimiter = "|", includedFields = Array(0, 1, 4, 7))
}
}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
index f068ed34727..f5e53c25293 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.relational
import org.apache.flink.api.java.utils.ParameterTool
@@ -24,8 +23,8 @@ import org.apache.flink.examples.java.relational.util.WebLogData
import org.apache.flink.util.Collector
/**
- * This program processes web logs and relational data.
- * It implements the following relational query:
+ * This program processes web logs and relational data. It implements the following relational
+ * query:
*
* {{{
* SELECT
@@ -44,11 +43,10 @@ import org.apache.flink.util.Collector
* );
* }}}
*
- *
- * Input files are plain text CSV files using the pipe character ('|') as field separator.
- * The tables referenced in the query can be generated using the
- * [org.apache.flink.examples.java.relational.util.WebLogDataGenerator]] and
- * have the following schemas
+ * Input files are plain text CSV files using the pipe character ('|') as field separator. The
+ * tables referenced in the query can be generated using the
+ * [org.apache.flink.examples.java.relational.util.WebLogDataGenerator]] and have the following
+ * schemas
*
* {{{
* CREATE TABLE Documents (
@@ -72,7 +70,6 @@ import org.apache.flink.util.Collector
* duration INT );
* }}}
*
- *
* Usage
* {{{
* WebLogAnalysis --documents <path> --ranks <path> --visits <path> --output <path>
@@ -83,10 +80,9 @@ import org.apache.flink.util.Collector
*
* This example shows how to use:
*
- * - tuple data types
- * - projection and join projection
- * - the CoGroup transformation for an anti-join
- *
+ * - tuple data types
+ * - projection and join projection
+ * - the CoGroup transformation for an anti-join
*/
object WebLogAnalysis {
@@ -113,17 +109,23 @@ object WebLogAnalysis {
val filteredVisits = visits
.filter(visit => visit._2.substring(0, 4).toInt == 2007)
- val joinDocsRanks = filteredDocs.join(filteredRanks).where(0).equalTo(1) {
- (doc, rank) => rank
- }.withForwardedFieldsSecond("*")
-
- val result = joinDocsRanks.coGroup(filteredVisits).where(1).equalTo(0) {
- (
- ranks: Iterator[(Int, String, Int)],
- visits: Iterator[(String, String)],
- out: Collector[(Int, String, Int)]) =>
+ val joinDocsRanks = filteredDocs
+ .join(filteredRanks)
+ .where(0)
+ .equalTo(1)((doc, rank) => rank)
+ .withForwardedFieldsSecond("*")
+
+ val result = joinDocsRanks
+ .coGroup(filteredVisits)
+ .where(1)
+ .equalTo(0) {
+ (
+ ranks: Iterator[(Int, String, Int)],
+ visits: Iterator[(String, String)],
+ out: Collector[(Int, String, Int)]) =>
if (visits.isEmpty) for (rank <- ranks) out.collect(rank)
- }.withForwardedFieldsFirst("*")
+ }
+ .withForwardedFieldsFirst("*")
// emit result
if (params.has("output")) {
@@ -136,8 +138,9 @@ object WebLogAnalysis {
}
- private def getDocumentsDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(String, String)] = {
+ private def getDocumentsDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(String, String)] = {
if (params.has("documents")) {
env.readCsvFile[(String, String)](
params.get("documents"),
@@ -146,15 +149,16 @@ object WebLogAnalysis {
} else {
println("Executing WebLogAnalysis example with default documents data set.")
println("Use --documents to specify file input.")
- val documents = WebLogData.DOCUMENTS map {
+ val documents = WebLogData.DOCUMENTS.map {
case Array(x, y) => (x.asInstanceOf[String], y.asInstanceOf[String])
}
env.fromCollection(documents)
}
}
- private def getRanksDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(Int, String, Int)] = {
+ private def getRanksDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(Int, String, Int)] = {
if (params.has("ranks")) {
env.readCsvFile[(Int, String, Int)](
params.get("ranks"),
@@ -163,15 +167,16 @@ object WebLogAnalysis {
} else {
println("Executing WebLogAnalysis example with default ranks data set.")
println("Use --ranks to specify file input.")
- val ranks = WebLogData.RANKS map {
+ val ranks = WebLogData.RANKS.map {
case Array(x, y, z) => (x.asInstanceOf[Int], y.asInstanceOf[String], z.asInstanceOf[Int])
}
env.fromCollection(ranks)
}
}
- private def getVisitsDataSet(env: ExecutionEnvironment, params: ParameterTool):
- DataSet[(String, String)] = {
+ private def getVisitsDataSet(
+ env: ExecutionEnvironment,
+ params: ParameterTool): DataSet[(String, String)] = {
if (params.has("visits")) {
env.readCsvFile[(String, String)](
params.get("visits"),
@@ -180,7 +185,7 @@ object WebLogAnalysis {
} else {
println("Executing WebLogAnalysis example with default visits data set.")
println("Use --visits to specify file input.")
- val visits = WebLogData.VISITS map {
+ val visits = WebLogData.VISITS.map {
case Array(x, y) => (x.asInstanceOf[String], y.asInstanceOf[String])
}
env.fromCollection(visits)
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala
index 0f3d6c8e1ff..2ad9e304a05 100644
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala
+++ b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.examples.scala.wordcount
import org.apache.flink.api.java.utils.ParameterTool
@@ -23,8 +22,8 @@ import org.apache.flink.api.scala._
import org.apache.flink.examples.java.wordcount.util.WordCountData
/**
- * Implements the "WordCount" program that computes a simple word occurrence histogram
- * over text files.
+ * Implements the "WordCount" program that computes a simple word occurrence histogram over text
+ * files.
*
* The input is a plain text file with lines separated by newline characters.
*
@@ -41,7 +40,6 @@ import org.apache.flink.examples.java.wordcount.util.WordCountData
* - write a simple Flink program.
* - use Tuple data types.
* - write and use user-defined functions.
- *
*/
object WordCount {
@@ -63,8 +61,9 @@ object WordCount {
env.fromCollection(WordCountData.WORDS)
}
- val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } }
- .map { (_, 1) }
+ val counts = text
+ .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty))
+ .map((_, 1))
.groupBy(0)
.sum(1)
@@ -78,5 +77,3 @@ object WordCount {
}
}
-
-
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala
index 3270980b16b..385f13b4ecd 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncClient.scala
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.async
import java.util.concurrent.ThreadLocalRandom
+
import scala.concurrent.{ExecutionContext, Future}
/** A simple asynchronous client that simulates interacting with an unreliable external service. */
@@ -26,7 +26,8 @@ class AsyncClient {
def query(key: Int)(implicit executor: ExecutionContext): Future[String] = Future {
val sleep = (ThreadLocalRandom.current.nextFloat * 100).toLong
- try Thread.sleep(sleep) catch {
+ try Thread.sleep(sleep)
+ catch {
case e: InterruptedException =>
throw new RuntimeException("AsyncClient was interrupted", e)
}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala
index b711a35c23c..b55cb60553c 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/async/AsyncIOExample.scala
@@ -15,17 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.async
-
import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFunction}
import org.apache.flink.streaming.api.scala._
+import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFunction}
import org.apache.flink.streaming.examples.async.util.SimpleSource
import java.util.concurrent.TimeUnit
+
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.{Failure, Success}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala
index 4812ec41784..e51e977d35c 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/iteration/IterateExample.scala
@@ -15,12 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.iteration
import org.apache.flink.api.common.serialization.SimpleStringEncoder
-
-import java.util.Random
import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.api.scala._
import org.apache.flink.configuration.MemorySize
@@ -32,23 +29,23 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceCont
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import java.time.Duration
+import java.util.Random
/**
* Example illustrating iterations in Flink streaming.
*
- * The program sums up random numbers and counts additions
- * it performs to reach a specific threshold in an iterative streaming fashion.
+ * The program sums up random numbers and counts additions it performs to reach a specific threshold
+ * in an iterative streaming fashion.
*
* This example shows how to use:
*
- * - streaming iterations,
- * - buffer timeout to enhance latency,
- * - directed outputs.
- *
+ * - streaming iterations,
+ * - buffer timeout to enhance latency,
+ * - directed outputs.
*/
object IterateExample {
- private final val Bound = 100
+ final private val Bound = 100
def main(args: Array[String]): Unit = {
// Checking input parameters
@@ -63,18 +60,19 @@ object IterateExample {
// create input stream of integer pairs
val inputStream: DataStream[(Int, Int)] =
- if (params.has("input")) {
- // map a list of strings to integer pairs
- env.readTextFile(params.get("input")).map { value: String =>
- val record = value.substring(1, value.length - 1)
- val splitted = record.split(",")
- (Integer.parseInt(splitted(0)), Integer.parseInt(splitted(1)))
+ if (params.has("input")) {
+ // map a list of strings to integer pairs
+ env.readTextFile(params.get("input")).map {
+ value: String =>
+ val record = value.substring(1, value.length - 1)
+ val splitted = record.split(",")
+ (Integer.parseInt(splitted(0)), Integer.parseInt(splitted(1)))
+ }
+ } else {
+ println("Executing Iterate example with default input data set.")
+ println("Use --input to specify file input.")
+ env.addSource(new RandomFibonacciSource)
}
- } else {
- println("Executing Iterate example with default input data set.")
- println("Use --input to specify file input.")
- env.addSource(new RandomFibonacciSource)
- }
def withinBound(value: (Int, Int)) = value._1 < Bound && value._2 < Bound
@@ -87,8 +85,8 @@ object IterateExample {
.iterate(
(iteration: DataStream[(Int, Int, Int, Int, Int)]) => {
// calculates the next Fibonacci number and increment the counter
- val step = iteration.map(value =>
- (value._1, value._2, value._4, value._3 + value._4, value._5 + 1))
+ val step = iteration.map(
+ value => (value._1, value._2, value._4, value._3 + value._4, value._5 + 1))
// testing which tuple needs to be iterated again
val feedback = step.filter(value => withinBound(value._3, value._4))
// giving back the input pair and the counter
@@ -98,18 +96,24 @@ object IterateExample {
(feedback, output)
}
// timeout after 5 seconds
- , 5000L
+ ,
+ 5000L
)
if (params.has("output")) {
- numbers.sinkTo(FileSink.forRowFormat[((Int, Int), Int)](
- new Path(params.get("output")),
- new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ numbers
+ .sinkTo(
+ FileSink
+ .forRowFormat[((Int, Int), Int)](
+ new Path(params.get("output")),
+ new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
} else {
println("Printing result to stdout. Use --output to specify output path.")
@@ -123,9 +127,7 @@ object IterateExample {
// USER FUNCTIONS
// *************************************************************************
- /**
- * Generate BOUND number of random integer pairs from the range from 0 to BOUND/2
- */
+ /** Generate BOUND number of random integer pairs from the range from 0 to BOUND/2 */
private class RandomFibonacciSource extends SourceFunction[(Int, Int)] {
val rnd = new Random()
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
index 607ca04622f..d46f705ac1e 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.join
import org.apache.flink.api.java.utils.ParameterTool
@@ -26,11 +25,11 @@ import org.apache.flink.streaming.api.windowing.time.Time
/**
* Example illustrating a windowed stream join between two data streams.
*
- * The example works on two input streams with pairs (name, grade) and (name, salary)
- * respectively. It joins the streams based on "name" within a configurable window.
+ * The example works on two input streams with pairs (name, grade) and (name, salary) respectively.
+ * It joins the streams based on "name" within a configurable window.
*
- * The example uses a built-in sample data generator that generates
- * the streams of pairs at a configurable rate.
+ * The example uses a built-in sample data generator that generates the streams of pairs at a
+ * configurable rate.
*/
object WindowJoin {
@@ -55,8 +54,9 @@ object WindowJoin {
val rate = params.getLong("rate", 3)
println("Using windowSize=" + windowSize + ", data rate=" + rate)
- println("To customize example, use: WindowJoin " +
- "[--windowSize <window-size-in-millis>] [--rate <elements-per-second>]")
+ println(
+ "To customize example, use: WindowJoin " +
+ "[--windowSize <window-size-in-millis>] [--rate <elements-per-second>]")
// obtain execution environment, run this example in "ingestion time"
val env = StreamExecutionEnvironment.getExecutionEnvironment
@@ -79,16 +79,16 @@ object WindowJoin {
env.execute("Windowed Join Example")
}
-
def joinStreams(
grades: DataStream[Grade],
salaries: DataStream[Salary],
- windowSize: Long) : DataStream[Person] = {
+ windowSize: Long): DataStream[Person] = {
- grades.join(salaries)
+ grades
+ .join(salaries)
.where(_.name)
.equalTo(_.name)
.window(TumblingEventTimeWindows.of(Time.milliseconds(windowSize)))
- .apply { (g, s) => Person(g.name, g.grade, s.salary) }
+ .apply((g, s) => Person(g.name, g.grade, s.salary))
}
}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala
index 10c30c57b38..5a0e59ecd24 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoinSampleData.scala
@@ -15,45 +15,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.join
-import java.io.Serializable
-import java.util.Random
-
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.examples.utils.ThrottledIterator
import org.apache.flink.streaming.scala.examples.join.WindowJoin.{Grade, Salary}
+import java.io.Serializable
+import java.util.Random
+
import scala.collection.JavaConverters._
-/**
- * Sample data for the [[WindowJoin]] example.
- */
+/** Sample data for the [[WindowJoin]] example. */
object WindowJoinSampleData {
-
+
private[join] val NAMES = Array("tom", "jerry", "alice", "bob", "john", "grace")
private[join] val GRADE_COUNT = 5
private[join] val SALARY_MAX = 10000
- /**
- * Continuously generates (name, grade).
- */
+ /** Continuously generates (name, grade). */
def getGradeSource(env: StreamExecutionEnvironment, rate: Long): DataStream[Grade] = {
- env.fromCollection(new ThrottledIterator(new GradeSource().asJava, rate).asScala)
+ env.fromCollection(new ThrottledIterator(new GradeSource().asJava, rate).asScala)
}
- /**
- * Continuously generates (name, salary).
- */
+ /** Continuously generates (name, salary). */
def getSalarySource(env: StreamExecutionEnvironment, rate: Long): DataStream[Salary] = {
env.fromCollection(new ThrottledIterator(new SalarySource().asJava, rate).asScala)
}
-
+
// --------------------------------------------------------------------------
-
+
class GradeSource extends Iterator[Grade] with Serializable {
-
+
private[this] val rnd = new Random(hashCode())
def hasNext: Boolean = true
@@ -62,7 +55,7 @@ object WindowJoinSampleData {
Grade(NAMES(rnd.nextInt(NAMES.length)), rnd.nextInt(GRADE_COUNT) + 1)
}
}
-
+
class SalarySource extends Iterator[Salary] with Serializable {
private[this] val rnd = new Random(hashCode())
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala
index fbb98ede71d..63b7d2478ca 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketWindowWordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.socket
import org.apache.flink.api.java.utils.ParameterTool
@@ -25,10 +24,9 @@ import org.apache.flink.streaming.api.windowing.time.Time
/**
* Implements a streaming windowed version of the "WordCount" program.
- *
- * This program connects to a server socket and reads strings from the socket.
- * The easiest way to try this out is to open a text sever (at port 12345)
- * using the ''netcat'' tool via
+ *
+ * This program connects to a server socket and reads strings from the socket. The easiest way to
+ * try this out is to open a text sever (at port 12345) using the ''netcat'' tool via
* {{{
* nc -l 12345 on Linux or nc -l -p 12345 on Windows
* }}}
@@ -37,7 +35,7 @@ import org.apache.flink.streaming.api.windowing.time.Time
object SocketWindowWordCount {
/** Main program method */
- def main(args: Array[String]) : Unit = {
+ def main(args: Array[String]): Unit = {
// the host and the port to connect to
var hostname: String = "localhost"
@@ -52,25 +50,26 @@ object SocketWindowWordCount {
System.err.println("No port specified. Please run 'SocketWindowWordCount " +
"--hostname <hostname> --port <port>', where hostname (localhost by default) and port " +
"is the address of the text server")
- System.err.println("To start a simple text server, run 'netcat -l <port>' " +
- "and type the input text into the command line")
+ System.err.println(
+ "To start a simple text server, run 'netcat -l <port>' " +
+ "and type the input text into the command line")
return
}
}
-
+
// get the execution environment
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
-
+
// get input data by connecting to the socket
val text: DataStream[String] = env.socketTextStream(hostname, port, '\n')
- // parse the data, group it, window it, and aggregate the counts
+ // parse the data, group it, window it, and aggregate the counts
val windowCounts = text
- .flatMap { w => w.split("\\s") }
- .map { w => WordWithCount(w, 1) }
- .keyBy(_.word)
- .window(TumblingProcessingTimeWindows.of(Time.seconds(5)))
- .sum("count")
+ .flatMap(w => w.split("\\s"))
+ .map(w => WordWithCount(w, 1))
+ .keyBy(_.word)
+ .window(TumblingProcessingTimeWindows.of(Time.seconds(5)))
+ .sum("count")
// print the results with a single thread, rather than in parallel
windowCounts.print()
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala
index a982feb50b2..49e0f76dd54 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/GroupedProcessingTimeWindowExample.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
import org.apache.flink.api.scala._
@@ -27,8 +26,8 @@ import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeW
import org.apache.flink.streaming.api.windowing.time.Time
/**
- * An example of grouped stream windowing into sliding time windows.
- * This example uses [[RichParallelSourceFunction]] to generate a list of key-value pair.
+ * An example of grouped stream windowing into sliding time windows. This example uses
+ * [[RichParallelSourceFunction]] to generate a list of key-value pair.
*/
object GroupedProcessingTimeWindowExample {
@@ -47,9 +46,7 @@ object GroupedProcessingTimeWindowExample {
env.execute()
}
- /**
- * Parallel data source that serves a list of key-value pair.
- */
+ /** Parallel data source that serves a list of key-value pair. */
private class DataSource extends RichParallelSourceFunction[(Long, Long)] {
@volatile private var running = true
@@ -74,7 +71,7 @@ object GroupedProcessingTimeWindowExample {
}
val endTime = System.currentTimeMillis()
- println(s"Took ${endTime - startTime} msecs for ${numElements} values")
+ println(s"Took ${endTime - startTime} msecs for $numElements values")
}
override def cancel(): Unit = running = false
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala
index a5d0b7e497f..c537f316d66 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/SessionWindowing.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
import org.apache.flink.api.common.serialization.SimpleStringEncoder
@@ -35,8 +34,8 @@ import org.apache.flink.streaming.api.windowing.time.Time
import java.time.Duration
/**
- * An example of grouped stream windowing in session windows with session timeout of 3 msec.
- * A source fetches elements with key, timestamp, and count.
+ * An example of grouped stream windowing in session windows with session timeout of 3 msec. A
+ * source fetches elements with key, timestamp, and count.
*/
object SessionWindowing {
@@ -62,14 +61,15 @@ object SessionWindowing {
("c", 11L, 1)
)
- val source: DataStream[(String, Long, Int)] = env.addSource(
- new SourceFunction[(String, Long, Int)]() {
+ val source: DataStream[(String, Long, Int)] =
+ env.addSource(new SourceFunction[(String, Long, Int)]() {
override def run(ctx: SourceContext[(String, Long, Int)]): Unit = {
- input.foreach(value => {
- ctx.collectWithTimestamp(value, value._2)
- ctx.emitWatermark(new Watermark(value._2 - 1))
- })
+ input.foreach(
+ value => {
+ ctx.collectWithTimestamp(value, value._2)
+ ctx.emitWatermark(new Watermark(value._2 - 1))
+ })
ctx.emitWatermark(new Watermark(Long.MaxValue))
}
@@ -84,14 +84,19 @@ object SessionWindowing {
.sum(2)
if (fileOutput) {
- aggregated.sinkTo(FileSink.forRowFormat[(String, Long, Int)](
- new Path(params.get("output")),
- new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ aggregated
+ .sinkTo(
+ FileSink
+ .forRowFormat[(String, Long, Int)](
+ new Path(params.get("output")),
+ new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
} else {
print("Printing result to stdout. Use --output to specify output path.")
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index d487488cfcb..4a0d84e06cd 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -15,10 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
-
import org.apache.flink.api.common.eventtime.WatermarkStrategy
import org.apache.flink.api.common.serialization.SimpleStringEncoder
import org.apache.flink.api.java.utils.ParameterTool
@@ -42,12 +40,10 @@ import java.time.Duration
import java.util.concurrent.TimeUnit
/**
- * An example of grouped stream windowing where different eviction and
- * trigger policies can be used. A source fetches events from cars
- * every 100 msec containing their id, their current speed (kmh),
- * overall elapsed distance (m) and a timestamp. The streaming
- * example triggers the top speed of each car every x meters elapsed
- * for the last y seconds.
+ * An example of grouped stream windowing where different eviction and trigger policies can be used.
+ * A source fetches events from cars every 100 msec containing their id, their current speed (kmh),
+ * overall elapsed distance (m) and a timestamp. The streaming example triggers the top speed of
+ * each car every x meters elapsed for the last y seconds.
*/
object TopSpeedWindowing {
@@ -97,11 +93,12 @@ object TopSpeedWindowing {
case Some(input) =>
// Create a new file source that will read files from a given set of directories.
// Each file will be processed as plain text and split based on newlines.
- val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input:_*)
- params.discoveryInterval.foreach { duration =>
- // If a discovery interval is provided, the source will
- // continuously watch the given directories for new files.
- builder.monitorContinuously(duration)
+ val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input: _*)
+ params.discoveryInterval.foreach {
+ duration =>
+ // If a discovery interval is provided, the source will
+ // continuously watch the given directories for new files.
+ builder.monitorContinuously(duration)
}
env
.fromSource(builder.build(), WatermarkStrategy.noWatermarks(), "file-input")
@@ -112,13 +109,17 @@ object TopSpeedWindowing {
}
val topSpeeds = cars
- .assignAscendingTimestamps( _.time )
+ .assignAscendingTimestamps(_.time)
.keyBy(_.carId)
.window(GlobalWindows.create)
.evictor(TimeEvictor.of(Time.of(evictionSec * 1000, TimeUnit.MILLISECONDS)))
- .trigger(DeltaTrigger.of(triggerMeters, new DeltaFunction[CarEvent] {
- def getDelta(oldSp: CarEvent, newSp: CarEvent): Double = newSp.distance - oldSp.distance
- }, cars.dataType.createSerializer(env.getConfig)))
+ .trigger(DeltaTrigger.of(
+ triggerMeters,
+ new DeltaFunction[CarEvent] {
+ def getDelta(oldSp: CarEvent, newSp: CarEvent): Double = newSp.distance - oldSp.distance
+ },
+ cars.dataType.createSerializer(env.getConfig)
+ ))
// .window(Time.of(evictionSec * 1000, (car : CarEvent) => car.time))
// .every(Delta.of[CarEvent](triggerMeters,
// (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0)))
@@ -129,12 +130,17 @@ object TopSpeedWindowing {
// Given an output directory, Flink will write the results to a file
// using a simple string encoding. In a production environment, this might
// be something more structured like CSV, Avro, JSON, or Parquet.
- topSpeeds.sinkTo(FileSink.forRowFormat[CarEvent](output, new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ topSpeeds
+ .sinkTo(
+ FileSink
+ .forRowFormat[CarEvent](output, new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
case None => topSpeeds.print().name("print-sink")
@@ -148,7 +154,7 @@ object TopSpeedWindowing {
// USER FUNCTIONS
// *************************************************************************
- def parseMap(line : String): CarEvent = {
+ def parseMap(line: String): CarEvent = {
val record = line.substring(1, line.length - 1).split(",")
CarEvent(record(0).toInt, record(1).toInt, record(2).toDouble, record(3).toLong)
}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala
index 6ba8810a194..ab1e1ce4a1e 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowWordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing
import org.apache.flink.api.common.eventtime.WatermarkStrategy
@@ -47,15 +46,13 @@ import java.time.Duration
* --slide <n>
* }}}
*
- * If no parameters are provided, the program is run with default data from
- * [[WordCountData]].
+ * If no parameters are provided, the program is run with default data from [[WordCountData]].
*
* This example shows how to:
*
- * - write a simple Flink Streaming program,
- * - use tuple data types,
- * - use basic windowing abstractions.
- *
+ * - write a simple Flink Streaming program,
+ * - use tuple data types,
+ * - use basic windowing abstractions.
*/
object WindowWordCount {
@@ -95,15 +92,16 @@ object WindowWordCount {
case Some(input) =>
// Create a new file source that will read files from a given set of directories.
// Each file will be processed as plain text and split based on newlines.
- val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input:_*)
- params.discoveryInterval.foreach { duration =>
- // If a discovery interval is provided, the source will
- // continuously watch the given directories for new files.
- builder.monitorContinuously(duration)
+ val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input: _*)
+ params.discoveryInterval.foreach {
+ duration =>
+ // If a discovery interval is provided, the source will
+ // continuously watch the given directories for new files.
+ builder.monitorContinuously(duration)
}
env.fromSource(builder.build(), WatermarkStrategy.noWatermarks(), "file-input")
case None =>
- env.fromElements(WordCountData.WORDS:_*).name("in-memory-input")
+ env.fromElements(WordCountData.WORDS: _*).name("in-memory-input")
}
val windowSize = params.getInt("window").getOrElse(250)
@@ -113,7 +111,8 @@ object WindowWordCount {
// The text lines read from the source are split into words
// using a user-defined function. The tokenizer, implemented below,
// will output each words as a (2-tuple) containing (word, 1)
- text.flatMap(new Tokenizer)
+ text
+ .flatMap(new Tokenizer)
.name("tokenizer")
// keyBy groups tuples based on the "_1" field, the word.
// Using a keyBy allows performing aggregations and other
@@ -134,12 +133,17 @@ object WindowWordCount {
// Given an output directory, Flink will write the results to a file
// using a simple string encoding. In a production environment, this might
// be something more structured like CSV, Avro, JSON, or Parquet.
- counts.sinkTo(FileSink.forRowFormat[(String, Int)](output, new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ counts
+ .sinkTo(
+ FileSink
+ .forRowFormat[(String, Int)](output, new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
case None => counts.print().name("print-sink")
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala
index 8f1d4c4d3a5..d216f249f5d 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/util/CarSource.scala
@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.windowing.util
+import org.apache.flink.api.java.tuple.{Tuple4 => JTuple4}
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.examples.windowing.util.{CarSource => JCarSource}
import org.apache.flink.streaming.scala.examples.windowing.TopSpeedWindowing.CarEvent
-import org.apache.flink.api.java.tuple.{Tuple4 => JTuple4}
-import java.lang.{Integer => JInt, Double => JDouble, Long => JLong}
+import java.lang.{Double => JDouble, Integer => JInt, Long => JLong}
/** A simple in-memory source. */
object CarSource {
@@ -48,8 +47,8 @@ private class WrappingCollector(ctx: SourceFunction.SourceContext[CarEvent])
ctx.collect(CarEvent(element.f0, element.f1, element.f2, element.f3))
override def collectWithTimestamp(
- element: JTuple4[JInt, JInt, JDouble, JLong],
- timestamp: Long): Unit =
+ element: JTuple4[JInt, JInt, JDouble, JLong],
+ timestamp: Long): Unit =
ctx.collectWithTimestamp(CarEvent(element.f0, element.f1, element.f2, element.f3), timestamp)
override def emitWatermark(mark: Watermark): Unit = ctx.emitWatermark(mark)
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala
index 26004028952..c19aa3ff2dc 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/WordCount.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.wordcount
import org.apache.flink.api.common.eventtime.WatermarkStrategy
@@ -41,25 +40,23 @@ import java.time.Duration
*
* Usage:
*
- * {{{ --input <path> }}} A list of input files and / or directories to read.
- * If no input is provided, the program is run with default data from [[WordCountData]].
+ * {{{--input <path>}}} A list of input files and / or directories to read. If no input is provided,
+ * the program is run with default data from [[WordCountData]].
*
- * {{{--discovery-interval <duration> }}} Turns the file reader
- * into a continuous source that will monitor the provided input directories
- * every interval and read any new files.
+ * {{{--discovery-interval <duration>}}} Turns the file reader into a continuous source that will
+ * monitor the provided input directories every interval and read any new files.
*
- * {{{--output <path> }}} The output directory where the Job will
- * write the results. If no output path is provided, the Job will print the results
- * to `stdout`
+ * {{{--output <path>}}} The output directory where the Job will write the results. If no output
+ * path is provided, the Job will print the results to `stdout`
*
- * {{{--execution-mode <mode> }}} The execution mode (BATCH, STREAMING, or AUTOMATIC) of this
+ * {{{--execution-mode <mode>}}} The execution mode (BATCH, STREAMING, or AUTOMATIC) of this
* pipeline.
*
* This example shows how to:
*
- * - Write a simple Flink DataStream program
- * - Use tuple data types
- * - Write and use a user-defined function
+ * - Write a simple Flink DataStream program
+ * - Use tuple data types
+ * - Write and use a user-defined function
*/
object WordCount {
@@ -104,22 +101,24 @@ object WordCount {
case Some(input) =>
// Create a new file source that will read files from a given set of directories.
// Each file will be processed as plain text and split based on newlines.
- val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input:_*)
- params.discoveryInterval.foreach { duration =>
- // If a discovery interval is provided, the source will
- // continuously watch the given directories for new files.
- builder.monitorContinuously(duration)
+ val builder = FileSource.forRecordStreamFormat(new TextLineInputFormat, input: _*)
+ params.discoveryInterval.foreach {
+ duration =>
+ // If a discovery interval is provided, the source will
+ // continuously watch the given directories for new files.
+ builder.monitorContinuously(duration)
}
env.fromSource(builder.build(), WatermarkStrategy.noWatermarks(), "file-input")
case None =>
- env.fromElements(WordCountData.WORDS:_*).name("in-memory-input")
+ env.fromElements(WordCountData.WORDS: _*).name("in-memory-input")
}
val counts =
// The text lines read from the source are split into words
// using a user-defined function. The tokenizer, implemented below,
// will output each word as a (2-tuple) containing (word, 1)
- text.flatMap(new Tokenizer)
+ text
+ .flatMap(new Tokenizer)
.name("tokenizer")
// keyBy groups tuples based on the "_1" field, the word.
// Using a keyBy allows performing aggregations and other
@@ -138,12 +137,17 @@ object WordCount {
// Given an output directory, Flink will write the results to a file
// using a simple string encoding. In a production environment, this might
// be something more structured like CSV, Avro, JSON, or Parquet.
- counts.sinkTo(FileSink.forRowFormat[(String, Int)](output, new SimpleStringEncoder())
- .withRollingPolicy(DefaultRollingPolicy.builder()
- .withMaxPartSize(MemorySize.ofMebiBytes(1))
- .withRolloverInterval(Duration.ofSeconds(10))
- .build())
- .build())
+ counts
+ .sinkTo(
+ FileSink
+ .forRowFormat[(String, Int)](output, new SimpleStringEncoder())
+ .withRollingPolicy(
+ DefaultRollingPolicy
+ .builder()
+ .withMaxPartSize(MemorySize.ofMebiBytes(1))
+ .withRolloverInterval(Duration.ofSeconds(10))
+ .build())
+ .build())
.name("file-sink")
case None => counts.print().name("print-sink")
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala
index f7f7b0b8d7d..cff44ba2897 100644
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/wordcount/util/CLI.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples.wordcount.util
import org.apache.flink.api.common.{ExecutionConfig, RuntimeExecutionMode}
diff --git a/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala b/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
index d4fcb5e840f..16fce22fb36 100644
--- a/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
+++ b/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
@@ -15,12 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.streaming.scala.examples
-import java.io.File
-
-import org.apache.commons.io.FileUtils
import org.apache.flink.core.fs.FileSystem.WriteMode
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala._
@@ -33,11 +29,13 @@ import org.apache.flink.streaming.scala.examples.wordcount.WordCount
import org.apache.flink.streaming.test.examples.join.WindowJoinData
import org.apache.flink.test.testdata.WordCountData
import org.apache.flink.test.util.{AbstractTestBase, TestBaseUtils}
+
+import org.apache.commons.io.FileUtils
import org.junit.Test
-/**
- * Integration test for streaming programs in Scala examples.
- */
+import java.io.File
+
+/** Integration test for streaming programs in Scala examples. */
class StreamingExamplesITCase extends AbstractTestBase {
@Test
@@ -48,10 +46,13 @@ class StreamingExamplesITCase extends AbstractTestBase {
// the example is inherently non-deterministic. The iteration timeout of 5000 ms
// is frequently not enough to make the test run stable on CI infrastructure
// with very small containers, so we cannot do a validation here
- IterateExample.main(Array(
- "--input", inputPath,
- "--output", resultPath
- ))
+ IterateExample.main(
+ Array(
+ "--input",
+ inputPath,
+ "--output",
+ resultPath
+ ))
}
@Test
@@ -63,31 +64,34 @@ class StreamingExamplesITCase extends AbstractTestBase {
val grades = env
.fromCollection(WindowJoinData.GRADES_INPUT.split("\n"))
- .map( line => {
- val fields = line.split(",")
- Grade(fields(1), fields(2).toInt)
- })
+ .map(
+ line => {
+ val fields = line.split(",")
+ Grade(fields(1), fields(2).toInt)
+ })
val salaries = env
.fromCollection(WindowJoinData.SALARIES_INPUT.split("\n"))
- .map( line => {
- val fields = line.split(",")
- Salary(fields(1), fields(2).toInt)
- })
-
- WindowJoin.joinStreams(grades, salaries, 100)
+ .map(
+ line => {
+ val fields = line.split(",")
+ Salary(fields(1), fields(2).toInt)
+ })
+
+ WindowJoin
+ .joinStreams(grades, salaries, 100)
.writeAsText(resultPath, WriteMode.OVERWRITE)
env.execute()
TestBaseUtils.checkLinesAgainstRegexp(resultPath, "^Person\\([a-z]+,(\\d),(\\d)+\\)")
- }
- finally try
- FileUtils.deleteDirectory(new File(resultPath))
+ } finally
+ try
+ FileUtils.deleteDirectory(new File(resultPath))
- catch {
- case _: Throwable =>
- }
+ catch {
+ case _: Throwable =>
+ }
}
@Test
@@ -103,13 +107,19 @@ class StreamingExamplesITCase extends AbstractTestBase {
val textPath = createTempFile("text.txt", WordCountData.TEXT)
val resultPath = getTempDirPath("result")
- WindowWordCount.main(Array(
- "--input", textPath,
- "--output", resultPath,
- "--window", windowSize,
- "--slide", slideSize,
- "--execution-mode", "AUTOMATIC"
- ))
+ WindowWordCount.main(
+ Array(
+ "--input",
+ textPath,
+ "--output",
+ resultPath,
+ "--window",
+ windowSize,
+ "--slide",
+ slideSize,
+ "--execution-mode",
+ "AUTOMATIC"
+ ))
// since the parallel tokenizers might have different speed
// the exact output can not be checked just whether it is well-formed
@@ -122,14 +132,16 @@ class StreamingExamplesITCase extends AbstractTestBase {
val textPath = createTempFile("text.txt", WordCountData.TEXT)
val resultPath = getTempDirPath("result")
- WordCount.main(Array(
- "--input", textPath,
- "--output", resultPath,
- "--execution-mode", "automatic"
- ))
-
- TestBaseUtils.compareResultsByLinesInMemory(
- WordCountData.COUNTS_AS_TUPLES,
- resultPath)
+ WordCount.main(
+ Array(
+ "--input",
+ textPath,
+ "--output",
+ resultPath,
+ "--execution-mode",
+ "automatic"
+ ))
+
+ TestBaseUtils.compareResultsByLinesInMemory(WordCountData.COUNTS_AS_TUPLES, resultPath)
}
}
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala
index 11c81b90792..2b18b27be3a 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/GettingStartedExample.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.table.examples.scala.basics
import org.apache.flink.table.api.{EnvironmentSettings, TableEnvironment, _}
@@ -31,12 +30,13 @@ import scala.collection.JavaConverters._
* Example for getting started with the Table & SQL API in Scala.
*
* The example shows how to create, transform, and query a table. It should give a first impression
- * about the look-and-feel of the API without going too much into details. See the other examples for
- * using connectors or more complex operations.
+ * about the look-and-feel of the API without going too much into details. See the other examples
+ * for using connectors or more complex operations.
*
* In particular, the example shows how to
* - setup a [[TableEnvironment]],
- * - use the environment for creating example tables, registering views, and executing SQL queries,
+ * - use the environment for creating example tables, registering views, and executing SQL
+ * queries,
* - transform tables with filters and projections,
* - declare user-defined functions,
* - and print/collect results locally.
@@ -49,19 +49,80 @@ object GettingStartedExample {
// setup the unified API
// in this case: declare that the table programs should be executed in batch mode
- val settings = EnvironmentSettings.newInstance()
+ val settings = EnvironmentSettings
+ .newInstance()
.inBatchMode()
.build()
val env = TableEnvironment.create(settings)
// create a table with example data without a connector required
val rawCustomers = env.fromValues(
- row("Guillermo Smith", LocalDate.parse("1992-12-12"), "4081 Valley Road", "08540", "New Jersey", "m", true, 0, 78, 3),
- row("Valeria Mendoza", LocalDate.parse("1970-03-28"), "1239 Rainbow Road", "90017", "Los Angeles", "f", true, 9, 39, 0),
- row("Leann Holloway", LocalDate.parse("1989-05-21"), "2359 New Street", "97401", "Eugene", null, true, null, null, null),
- row("Brandy Sanders", LocalDate.parse("1956-05-26"), "4891 Walkers-Ridge-Way", "73119", "Oklahoma City", "m", false, 9, 39, 0),
- row("John Turner", LocalDate.parse("1982-10-02"), "2359 New Street", "60605", "Chicago", "m", true, 12, 39, 0),
- row("Ellen Ortega", LocalDate.parse("1985-06-18"), "2448 Rodney STreet", "85023", "Phoenix", "f", true, 0, 78, 3)
+ row(
+ "Guillermo Smith",
+ LocalDate.parse("1992-12-12"),
+ "4081 Valley Road",
+ "08540",
+ "New Jersey",
+ "m",
+ true,
+ 0,
+ 78,
+ 3),
+ row(
+ "Valeria Mendoza",
+ LocalDate.parse("1970-03-28"),
+ "1239 Rainbow Road",
+ "90017",
+ "Los Angeles",
+ "f",
+ true,
+ 9,
+ 39,
+ 0),
+ row(
+ "Leann Holloway",
+ LocalDate.parse("1989-05-21"),
+ "2359 New Street",
+ "97401",
+ "Eugene",
+ null,
+ true,
+ null,
+ null,
+ null),
+ row(
+ "Brandy Sanders",
+ LocalDate.parse("1956-05-26"),
+ "4891 Walkers-Ridge-Way",
+ "73119",
+ "Oklahoma City",
+ "m",
+ false,
+ 9,
+ 39,
+ 0),
+ row(
+ "John Turner",
+ LocalDate.parse("1982-10-02"),
+ "2359 New Street",
+ "60605",
+ "Chicago",
+ "m",
+ true,
+ 12,
+ 39,
+ 0),
+ row(
+ "Ellen Ortega",
+ LocalDate.parse("1985-06-18"),
+ "2448 Rodney STreet",
+ "85023",
+ "Phoenix",
+ "f",
+ true,
+ 0,
+ 78,
+ 3)
)
// handle ranges of columns easily
@@ -78,12 +139,11 @@ object GettingStartedExample {
// call execute() and print() to get insights
env
.sqlQuery("""
- |SELECT
- | COUNT(*) AS `number of customers`,
- | AVG(YEAR(date_of_birth)) AS `average birth year`
- |FROM `customers`
- |""".stripMargin
- )
+ |SELECT
+ | COUNT(*) AS `number of customers`,
+ | AVG(YEAR(date_of_birth)) AS `average birth year`
+ |FROM `customers`
+ |""".stripMargin)
.execute()
.print()
@@ -108,7 +168,10 @@ object GettingStartedExample {
val actualOutput = iterator.asScala.toSet
val expectedOutput = Set(
- Row.of("GUILLERMO SMITH", LocalDate.parse("1992-12-12"), "4081 VALLEY ROAD, 08540, NEW JERSEY"),
+ Row.of(
+ "GUILLERMO SMITH",
+ LocalDate.parse("1992-12-12"),
+ "4081 VALLEY ROAD, 08540, NEW JERSEY"),
Row.of("JOHN TURNER", LocalDate.parse("1982-10-02"), "2359 NEW STREET, 60605, CHICAGO"),
Row.of("ELLEN ORTEGA", LocalDate.parse("1985-06-18"), "2448 RODNEY STREET, 85023, PHOENIX")
)
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala
index 3eb02119a63..6bfb061b0f3 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamSQLExample.scala
@@ -47,15 +47,11 @@ object StreamSQLExample {
// set up the Scala Table API
val tableEnv = StreamTableEnvironment.create(env)
- val orderA = env.fromCollection(Seq(
- Order(1L, "beer", 3),
- Order(1L, "diaper", 4),
- Order(3L, "rubber", 2)))
+ val orderA =
+ env.fromCollection(Seq(Order(1L, "beer", 3), Order(1L, "diaper", 4), Order(3L, "rubber", 2)))
- val orderB = env.fromCollection(Seq(
- Order(2L, "pen", 3),
- Order(2L, "rubber", 3),
- Order(4L, "beer", 1)))
+ val orderB =
+ env.fromCollection(Seq(Order(2L, "pen", 3), Order(2L, "rubber", 3), Order(4L, "beer", 1)))
// convert the first DataStream to a Table object
// it will be used "inline" and is not registered in a catalog
@@ -66,11 +62,10 @@ object StreamSQLExample {
tableEnv.createTemporaryView("TableB", orderB)
// union the two tables
- val result = tableEnv.sqlQuery(
- s"""
- |SELECT * FROM $tableA WHERE amount > 2
- |UNION ALL
- |SELECT * FROM TableB WHERE amount < 2
+ val result = tableEnv.sqlQuery(s"""
+ |SELECT * FROM $tableA WHERE amount > 2
+ |UNION ALL
+ |SELECT * FROM TableB WHERE amount < 2
""".stripMargin)
// convert the Table back to an insert-only DataStream of type `Order`
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala
index bf87231c38f..1e71f8e52af 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/StreamTableExample.scala
@@ -23,12 +23,12 @@ import org.apache.flink.table.api._
import org.apache.flink.table.api.bridge.scala._
/**
- * Simple example for demonstrating the use of Table API on a Stream Table.
- *
- * This example shows how to:
- * - Convert DataStreams to Tables
- * - Apply union, select, and filter operations
- */
+ * Simple example for demonstrating the use of Table API on a Stream Table.
+ *
+ * This example shows how to:
+ * - Convert DataStreams to Tables
+ * - Apply union, select, and filter operations
+ */
object StreamTableExample {
// *************************************************************************
@@ -41,18 +41,17 @@ object StreamTableExample {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tEnv = StreamTableEnvironment.create(env)
- val orderA = env.fromCollection(Seq(
- Order(1L, "beer", 3),
- Order(1L, "diaper", 4),
- Order(3L, "rubber", 2))).toTable(tEnv)
+ val orderA = env
+ .fromCollection(Seq(Order(1L, "beer", 3), Order(1L, "diaper", 4), Order(3L, "rubber", 2)))
+ .toTable(tEnv)
- val orderB = env.fromCollection(Seq(
- Order(2L, "pen", 3),
- Order(2L, "rubber", 3),
- Order(4L, "beer", 1))).toTable(tEnv)
+ val orderB = env
+ .fromCollection(Seq(Order(2L, "pen", 3), Order(2L, "rubber", 3), Order(4L, "beer", 1)))
+ .toTable(tEnv)
// union the two tables
- val result: DataStream[Order] = orderA.unionAll(orderB)
+ val result: DataStream[Order] = orderA
+ .unionAll(orderB)
.select('user, 'product, 'amount)
.where('amount > 2)
.toDataStream(classOf[Order])
diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala
index 7d220a6e6f4..42a772c66c9 100644
--- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/basics/WordCountSQLExample.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.table.examples.scala.basics
import org.apache.flink.table.api._
@@ -26,23 +25,25 @@ object WordCountSQLExample {
def main(args: Array[String]): Unit = {
// set up the Table API
- val settings = EnvironmentSettings.newInstance()
+ val settings = EnvironmentSettings
+ .newInstance()
.inBatchMode()
.build()
val tableEnv = TableEnvironment.create(settings)
// execute a Flink SQL job and print the result locally
- tableEnv.executeSql(
- // define the aggregation
- "SELECT word, SUM(frequency) AS `count`\n"
+ tableEnv
+ .executeSql(
+ // define the aggregation
+ "SELECT word, SUM(frequency) AS `count`\n"
// read from an artificial fixed-size table with rows and columns
- + "FROM (\n"
- + " VALUES ('Hello', 1), ('Ciao', 1), ('Hello', 2)\n"
- + ")\n"
- // name the table and its columns
- + "AS WordTable(word, frequency)\n"
- // group for aggregation
- + "GROUP BY word")
+ + "FROM (\n"
+ + " VALUES ('Hello', 1), ('Ciao', 1), ('Hello', 2)\n"
+ + ")\n"
+ // name the table and its columns
+ + "AS WordTable(word, frequency)\n"
+ // group for aggregation
+ + "GROUP BY word")
.print()
}
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala
index 565da3178fe..53edb6a2323 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/CEP.scala
@@ -17,45 +17,50 @@
*/
package org.apache.flink.cep.scala
+import org.apache.flink.cep.{CEP => JCEP, EventComparator}
import org.apache.flink.cep.scala.pattern.Pattern
-import org.apache.flink.cep.{EventComparator, CEP => JCEP}
import org.apache.flink.streaming.api.scala.DataStream
-/**
- * Utility method to transform a [[DataStream]] into a [[PatternStream]] to do CEP.
- */
+/** Utility method to transform a [[DataStream]] into a [[PatternStream]] to do CEP. */
object CEP {
+
/**
- * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API.
- * See [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying
- * Java API works.
- *
- * @param input DataStream containing the input events
- * @param pattern Pattern specification which shall be detected
- * @tparam T Type of the input events
- * @return Resulting pattern stream
- */
+ * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API. See
+ * [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying Java API works.
+ *
+ * @param input
+ * DataStream containing the input events
+ * @param pattern
+ * Pattern specification which shall be detected
+ * @tparam T
+ * Type of the input events
+ * @return
+ * Resulting pattern stream
+ */
def pattern[T](input: DataStream[T], pattern: Pattern[T, _ <: T]): PatternStream[T] = {
wrapPatternStream(JCEP.pattern(input.javaStream, pattern.wrappedPattern))
}
/**
- * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API.
- * See [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying
- * Java API works.
- *
- * @param input DataStream containing the input events
- * @param pattern Pattern specification which shall be detected
- * @param comparator Comparator to sort events with equal timestamps
- * @tparam T Type of the input events
- * @return Resulting pattern stream
- */
+ * Transforms a [[DataStream]] into a [[PatternStream]] in the Scala API. See
+ * [[org.apache.flink.cep.CEP}]]for a more detailed description how the underlying Java API works.
+ *
+ * @param input
+ * DataStream containing the input events
+ * @param pattern
+ * Pattern specification which shall be detected
+ * @param comparator
+ * Comparator to sort events with equal timestamps
+ * @tparam T
+ * Type of the input events
+ * @return
+ * Resulting pattern stream
+ */
def pattern[T](
- input: DataStream[T],
- pattern: Pattern[T, _ <: T],
- comparator: EventComparator[T]): PatternStream[T] = {
+ input: DataStream[T],
+ pattern: Pattern[T, _ <: T],
+ comparator: EventComparator[T]): PatternStream[T] = {
wrapPatternStream(JCEP.pattern(input.javaStream, pattern.wrappedPattern, comparator))
}
}
-
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
index ebc1c27c8ea..215a914d59a 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
@@ -17,168 +17,187 @@
*/
package org.apache.flink.cep.scala
-import java.util.{UUID, List => JList, Map => JMap}
-
import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.cep.{PatternFlatSelectFunction, PatternFlatTimeoutFunction, PatternSelectFunction, PatternStream => JPatternStream, PatternTimeoutFunction}
import org.apache.flink.cep.functions.PatternProcessFunction
-import org.apache.flink.cep.{PatternFlatSelectFunction, PatternFlatTimeoutFunction, PatternSelectFunction, PatternTimeoutFunction, PatternStream => JPatternStream}
import org.apache.flink.streaming.api.scala.{asScalaStream, _}
import org.apache.flink.util.Collector
+import java.util.{List => JList, Map => JMap, UUID}
+
import scala.collection.Map
/**
- * Stream abstraction for CEP pattern detection. A pattern stream is a stream which emits detected
- * pattern sequences as a map of events associated with their names. The pattern is detected using
- * a [[org.apache.flink.cep.nfa.NFA]]. In order to process the detected sequences, the user has to
- * specify a [[PatternSelectFunction]] or a [[PatternFlatSelectFunction]].
- *
- * @param jPatternStream Underlying pattern stream from Java API
- * @tparam T Type of the events
- */
+ * Stream abstraction for CEP pattern detection. A pattern stream is a stream which emits detected
+ * pattern sequences as a map of events associated with their names. The pattern is detected using a
+ * [[org.apache.flink.cep.nfa.NFA]]. In order to process the detected sequences, the user has to
+ * specify a [[PatternSelectFunction]] or a [[PatternFlatSelectFunction]].
+ *
+ * @param jPatternStream
+ * Underlying pattern stream from Java API
+ * @tparam T
+ * Type of the events
+ */
class PatternStream[T](jPatternStream: JPatternStream[T]) {
private[flink] def wrappedPatternStream = jPatternStream
/**
- * Applies a process function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternProcessFunction]] is called.
- *
- * @param patternProcessFunction The pattern process function which is called for each detected
- * pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern select function.
- */
- def process[R: TypeInformation](patternProcessFunction: PatternProcessFunction[T, R])
- : DataStream[R] = {
+ * Applies a process function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternProcessFunction]] is called.
+ *
+ * @param patternProcessFunction
+ * The pattern process function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern select function.
+ */
+ def process[R: TypeInformation](
+ patternProcessFunction: PatternProcessFunction[T, R]): DataStream[R] = {
asScalaStream(jPatternStream.process(patternProcessFunction, implicitly[TypeInformation[R]]))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern select function.
- */
- def select[R: TypeInformation](patternSelectFunction: PatternSelectFunction[T, R])
- : DataStream[R] = {
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern select function.
+ */
+ def select[R: TypeInformation](
+ patternSelectFunction: PatternSelectFunction[T, R]): DataStream[R] = {
asScalaStream(jPatternStream.select(patternSelectFunction, implicitly[TypeInformation[R]]))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting timeout event.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contains the resulting events and resulting timeout
- * events.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting timeout event.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contains the resulting events and resulting timeout events.
+ */
@deprecated
def select[L: TypeInformation, R: TypeInformation](
- patternTimeoutFunction: PatternTimeoutFunction[T, L],
- patternSelectFunction: PatternSelectFunction[T, R])
- : DataStream[Either[L, R]] = {
+ patternTimeoutFunction: PatternTimeoutFunction[T, L],
+ patternSelectFunction: PatternSelectFunction[T, R]): DataStream[Either[L, R]] = {
val outputTag = OutputTag[L](UUID.randomUUID().toString)
val mainStream = select(outputTag, patternTimeoutFunction, patternSelectFunction)
mainStream.connect(mainStream.getSideOutput[L](outputTag)).map(r => Right(r), l => Left(l))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting timeout event.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream which contains the resulting elements with the resulting timeout elements
- * in a side output.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting timeout event.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream which contains the resulting elements with the resulting timeout elements in a
+ * side output.
+ */
def select[L: TypeInformation, R: TypeInformation](
- outputTag: OutputTag[L],
- patternTimeoutFunction: PatternTimeoutFunction[T, L],
- patternSelectFunction: PatternSelectFunction[T, R])
- : DataStream[R] = {
+ outputTag: OutputTag[L],
+ patternTimeoutFunction: PatternTimeoutFunction[T, L],
+ patternSelectFunction: PatternSelectFunction[T, R]): DataStream[R] = {
val cleanedSelect = cleanClosure(patternSelectFunction)
val cleanedTimeout = cleanClosure(patternTimeoutFunction)
asScalaStream(
jPatternStream
- .select(outputTag, cleanedTimeout, implicitly[TypeInformation[R]], cleanedSelect)
+ .select(outputTag, cleanedTimeout, implicitly[TypeInformation[R]], cleanedSelect)
)
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern flat select
- * function.
- */
- def flatSelect[R: TypeInformation](patternFlatSelectFunction: PatternFlatSelectFunction[T, R])
- : DataStream[R] = {
- asScalaStream(jPatternStream
- .flatSelect(patternFlatSelectFunction, implicitly[TypeInformation[R]]))
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern flat select function.
+ */
+ def flatSelect[R: TypeInformation](
+ patternFlatSelectFunction: PatternFlatSelectFunction[T, R]): DataStream[R] = {
+ asScalaStream(
+ jPatternStream
+ .flatSelect(patternFlatSelectFunction, implicitly[TypeInformation[R]]))
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contains the resulting events and the resulting
- * timeout events wrapped in a [[Either]] type.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contains the resulting events and the resulting timeout
+ * events wrapped in a [[Either]] type.
+ */
@deprecated
def flatSelect[L: TypeInformation, R: TypeInformation](
- patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
- patternFlatSelectFunction: PatternFlatSelectFunction[T, R])
- : DataStream[Either[L, R]] = {
+ patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
+ patternFlatSelectFunction: PatternFlatSelectFunction[T, R]): DataStream[Either[L, R]] = {
val outputTag = OutputTag[L]("dummy-timeouted")
val mainStream = flatSelect(outputTag, patternFlatTimeoutFunction, patternFlatSelectFunction)
@@ -186,54 +205,57 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream which contains the resulting elements with the resulting timeout elements
- * in a side output.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream which contains the resulting elements with the resulting timeout elements in a
+ * side output.
+ */
def flatSelect[L: TypeInformation, R: TypeInformation](
- outputTag: OutputTag[L],
- patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
- patternFlatSelectFunction: PatternFlatSelectFunction[T, R])
- : DataStream[R] = {
+ outputTag: OutputTag[L],
+ patternFlatTimeoutFunction: PatternFlatTimeoutFunction[T, L],
+ patternFlatSelectFunction: PatternFlatSelectFunction[T, R]): DataStream[R] = {
val cleanedSelect = cleanClosure(patternFlatSelectFunction)
val cleanedTimeout = cleanClosure(patternFlatTimeoutFunction)
asScalaStream(
- jPatternStream.flatSelect(
- outputTag,
- cleanedTimeout,
- implicitly[TypeInformation[R]],
- cleanedSelect))
+ jPatternStream
+ .flatSelect(outputTag, cleanedTimeout, implicitly[TypeInformation[R]], cleanedSelect))
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
- * one resulting element.
- *
- * @param patternSelectFun The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern select function.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * @param patternSelectFun
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern select function.
+ */
def select[R: TypeInformation](patternSelectFun: Map[String, Iterable[T]] => R): DataStream[R] = {
val cleanFun = cleanClosure(patternSelectFun)
@@ -245,31 +267,34 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting element.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contain the resulting events and resulting timeout
- * events.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting element.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contain the resulting events and resulting timeout events.
+ */
@deprecated
def select[L: TypeInformation, R: TypeInformation](
- patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L) (
- patternSelectFunction: Map[String, Iterable[T]] => R)
- : DataStream[Either[L, R]] = {
+ patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L)(
+ patternSelectFunction: Map[String, Iterable[T]] => R): DataStream[Either[L, R]] = {
val cleanSelectFun = cleanClosure(patternSelectFunction)
val cleanTimeoutFun = cleanClosure(patternTimeoutFunction)
@@ -287,31 +312,34 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a select function to the detected pattern sequence. For each pattern sequence the
- * provided [[PatternSelectFunction]] is called. The pattern select function can produce
- * exactly one resulting element.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
- * timeout function has to produce exactly one resulting element.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternTimeoutFunction The pattern timeout function which is called for each partial
- * pattern sequence which has timed out.
- * @param patternSelectFunction The pattern select function which is called for each detected
- * pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream of either type which contain the resulting events and resulting timeout
- * events.
- */
+ * Applies a select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternSelectFunction]] is called. The pattern select function can produce exactly
+ * one resulting element.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternTimeoutFunction]] is called. The pattern
+ * timeout function has to produce exactly one resulting element.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternTimeoutFunction
+ * The pattern timeout function which is called for each partial pattern sequence which has
+ * timed out.
+ * @param patternSelectFunction
+ * The pattern select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream of either type which contain the resulting events and resulting timeout events.
+ */
def select[L: TypeInformation, R: TypeInformation](outputTag: OutputTag[L])(
- patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L) (
- patternSelectFunction: Map[String, Iterable[T]] => R)
- : DataStream[R] = {
+ patternTimeoutFunction: (Map[String, Iterable[T]], Long) => L)(
+ patternSelectFunction: Map[String, Iterable[T]] => R): DataStream[R] = {
val cleanSelectFun = cleanClosure(patternSelectFunction)
val cleanTimeoutFun = cleanClosure(patternTimeoutFunction)
@@ -329,18 +357,19 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function
- * can produce an arbitrary number of resulting elements.
- *
- * @param patternFlatSelectFun The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam R Type of the resulting elements
- * @return [[DataStream]] which contains the resulting elements from the pattern flat select
- * function.
- */
- def flatSelect[R: TypeInformation](patternFlatSelectFun: (Map[String, Iterable[T]],
- Collector[R]) => Unit): DataStream[R] = {
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * @param patternFlatSelectFun
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam R
+ * Type of the resulting elements
+ * @return
+ * [[DataStream]] which contains the resulting elements from the pattern flat select function.
+ */
+ def flatSelect[R: TypeInformation](
+ patternFlatSelectFun: (Map[String, Iterable[T]], Collector[R]) => Unit): DataStream[R] = {
val cleanFun = cleanClosure(patternFlatSelectFun)
val patternFlatSelectFunction: PatternFlatSelectFunction[T, R] =
@@ -353,31 +382,36 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
- *
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @deprecated Use the version that returns timeouted events as a side-output
- * @return Data stream of either type which contains the resulting events and the resulting
- * timeout events wrapped in a [[Either]] type.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * The resulting event and the resulting timeout event are wrapped in an [[Either]] instance.
+ *
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @deprecated
+ * Use the version that returns timeouted events as a side-output
+ * @return
+ * Data stream of either type which contains the resulting events and the resulting timeout
+ * events wrapped in a [[Either]] type.
+ */
@deprecated
def flatSelect[L: TypeInformation, R: TypeInformation](
- patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit) (
+ patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit)(
patternFlatSelectFunction: (Map[String, Iterable[T]], Collector[R]) => Unit)
- : DataStream[Either[L, R]] = {
+ : DataStream[Either[L, R]] = {
val cleanSelectFun = cleanClosure(patternFlatSelectFunction)
val cleanTimeoutFun = cleanClosure(patternFlatTimeoutFunction)
@@ -389,9 +423,9 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
val patternFlatTimeoutFun = new PatternFlatTimeoutFunction[T, L] {
override def timeout(
- pattern: JMap[String, JList[T]],
- timeoutTimestamp: Long, out: Collector[L])
- : Unit = {
+ pattern: JMap[String, JList[T]],
+ timeoutTimestamp: Long,
+ out: Collector[L]): Unit = {
cleanTimeoutFun(mapToScala(pattern), timeoutTimestamp, out)
}
}
@@ -400,31 +434,36 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
/**
- * Applies a flat select function to the detected pattern sequence. For each pattern sequence
- * the provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can
- * produce an arbitrary number of resulting elements.
- *
- * Additionally a timeout function is applied to partial event patterns which have timed out. For
- * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
- * pattern timeout function can produce an arbitrary number of resulting timeout events.
- *
- * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
- * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
- *
- * @param outputTag [[OutputTag]] that identifies side output with timeouted patterns
- * @param patternFlatTimeoutFunction The pattern flat timeout function which is called for each
- * partially matched pattern sequence which has timed out.
- * @param patternFlatSelectFunction The pattern flat select function which is called for each
- * detected pattern sequence.
- * @tparam L Type of the resulting timeout event
- * @tparam R Type of the resulting event
- * @return Data stream of either type which contains the resulting events and the resulting
- * timeout events wrapped in a [[Either]] type.
- */
+ * Applies a flat select function to the detected pattern sequence. For each pattern sequence the
+ * provided [[PatternFlatSelectFunction]] is called. The pattern flat select function can produce
+ * an arbitrary number of resulting elements.
+ *
+ * Additionally a timeout function is applied to partial event patterns which have timed out. For
+ * each partial pattern sequence the provided [[PatternFlatTimeoutFunction]] is called. The
+ * pattern timeout function can produce an arbitrary number of resulting timeout events.
+ *
+ * You can get the stream of timeouted matches using [[DataStream.getSideOutput()]] on the
+ * [[DataStream]] resulting from the windowed operation with the same [[OutputTag]].
+ *
+ * @param outputTag
+ * [[OutputTag]] that identifies side output with timeouted patterns
+ * @param patternFlatTimeoutFunction
+ * The pattern flat timeout function which is called for each partially matched pattern sequence
+ * which has timed out.
+ * @param patternFlatSelectFunction
+ * The pattern flat select function which is called for each detected pattern sequence.
+ * @tparam L
+ * Type of the resulting timeout event
+ * @tparam R
+ * Type of the resulting event
+ * @return
+ * Data stream of either type which contains the resulting events and the resulting timeout
+ * events wrapped in a [[Either]] type.
+ */
def flatSelect[L: TypeInformation, R: TypeInformation](outputTag: OutputTag[L])(
- patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit) (
- patternFlatSelectFunction: (Map[String, Iterable[T]], Collector[R]) => Unit)
- : DataStream[R] = {
+ patternFlatTimeoutFunction: (Map[String, Iterable[T]], Long, Collector[L]) => Unit)(
+ patternFlatSelectFunction: (Map[String, Iterable[T]], Collector[R]) => Unit)
+ : DataStream[R] = {
val cleanSelectFun = cleanClosure(patternFlatSelectFunction)
val cleanTimeoutFun = cleanClosure(patternFlatTimeoutFunction)
@@ -436,9 +475,9 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
val patternFlatTimeoutFun = new PatternFlatTimeoutFunction[T, L] {
override def timeout(
- pattern: JMap[String, JList[T]],
- timeoutTimestamp: Long, out: Collector[L])
- : Unit = {
+ pattern: JMap[String, JList[T]],
+ timeoutTimestamp: Long,
+ out: Collector[L]): Unit = {
cleanTimeoutFun(mapToScala(pattern), timeoutTimestamp, out)
}
}
@@ -446,7 +485,7 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
flatSelect(outputTag, patternFlatTimeoutFun, patternFlatSelectFun)
}
- def sideOutputLateData(lateDataOutputTag: OutputTag[T]): PatternStream[T] = {
+ def sideOutputLateData(lateDataOutputTag: OutputTag[T]): PatternStream[T] = {
PatternStream[T](jPatternStream.sideOutputLateData(lateDataOutputTag))
}
@@ -460,12 +499,15 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
}
object PatternStream {
+
/**
- *
- * @param jPatternStream Underlying pattern stream from Java API
- * @tparam T Type of the events
- * @return A new pattern stream wrapping the pattern stream from Java APU
- */
+ * @param jPatternStream
+ * Underlying pattern stream from Java API
+ * @tparam T
+ * Type of the events
+ * @return
+ * A new pattern stream wrapping the pattern stream from Java APU
+ */
def apply[T](jPatternStream: JPatternStream[T]): PatternStream[T] = {
new PatternStream[T](jPatternStream)
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala
index 8bb713397f9..55340c2ad3e 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/conditions/Context.scala
@@ -19,15 +19,15 @@ package org.apache.flink.cep.scala.conditions
import java.io.Serializable
-/**
- * The context used when evaluating the {@link IterativeCondition condition}.
- */
+/** The context used when evaluating the {@link IterativeCondition condition}. */
trait Context[T] extends Serializable {
+
/**
- * @return An { @link Iterable} over the already accepted elements
- * for a given pattern. Elements are iterated in the order they were
- * inserted in the pattern.
- * @param name The name of the pattern.
- */
+ * @return
+ * An { @link Iterable} over the already accepted elements for a given pattern. Elements are
+ * iterated in the order they were inserted in the pattern.
+ * @param name
+ * The name of the pattern.
+ */
def getEventsForPattern(name: String): Iterable[T]
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala
index 223e9c90c6f..52ffae2475f 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/package.scala
@@ -17,25 +17,28 @@
*/
package org.apache.flink.cep
-import java.util.{List => JList, Map => JMap}
-
import org.apache.flink.api.scala.ClosureCleaner
import org.apache.flink.cep.{PatternStream => JPatternStream}
+import java.util.{List => JList, Map => JMap}
+
package object scala {
import collection.JavaConverters._
import collection.Map
/**
- * Utility method to wrap [[org.apache.flink.cep.PatternStream]] for usage with the Scala API.
- *
- * @param javaPatternStream The underlying pattern stream from the Java API
- * @tparam T Type of the events
- * @return A pattern stream from the Scala API which wraps a pattern stream from the Java API
- */
- private[flink] def wrapPatternStream[T](javaPatternStream: JPatternStream[T])
- : scala.PatternStream[T] = {
+ * Utility method to wrap [[org.apache.flink.cep.PatternStream]] for usage with the Scala API.
+ *
+ * @param javaPatternStream
+ * The underlying pattern stream from the Java API
+ * @tparam T
+ * Type of the events
+ * @return
+ * A pattern stream from the Scala API which wraps a pattern stream from the Java API
+ */
+ private[flink] def wrapPatternStream[T](
+ javaPatternStream: JPatternStream[T]): scala.PatternStream[T] = {
Option(javaPatternStream) match {
case Some(p) => PatternStream[T](p)
case None =>
@@ -52,4 +55,3 @@ package object scala {
map.asScala.mapValues(_.asScala.toIterable)
}
}
-
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala
index 6210956bfb2..0143625e2d9 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/GroupPattern.scala
@@ -17,24 +17,27 @@
*/
package org.apache.flink.cep.scala.pattern
-import org.apache.flink.cep.pattern.conditions.IterativeCondition
import org.apache.flink.cep.pattern.{GroupPattern => JGroupPattern}
+import org.apache.flink.cep.pattern.conditions.IterativeCondition
/**
- * Base class for a group pattern definition.
- *
- * @param jGroupPattern Underlying Java API GroupPattern
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- */
-class GroupPattern[T , F <: T](jGroupPattern: JGroupPattern[T, F])
- extends Pattern[T , F](jGroupPattern) {
+ * Base class for a group pattern definition.
+ *
+ * @param jGroupPattern
+ * Underlying Java API GroupPattern
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ */
+class GroupPattern[T, F <: T](jGroupPattern: JGroupPattern[T, F])
+ extends Pattern[T, F](jGroupPattern) {
- override def where(condition: IterativeCondition[F] ) =
- throw new UnsupportedOperationException ("GroupPattern does not support where clause.")
+ override def where(condition: IterativeCondition[F]) =
+ throw new UnsupportedOperationException("GroupPattern does not support where clause.")
- override def or(condition: IterativeCondition[F] ) =
- throw new UnsupportedOperationException ("GroupPattern does not support or clause.")
+ override def or(condition: IterativeCondition[F]) =
+ throw new UnsupportedOperationException("GroupPattern does not support or clause.")
override def subtype[S <: F](clazz: Class[S]) =
throw new UnsupportedOperationException("GroupPattern does not support subtype clause.")
@@ -44,13 +47,17 @@ class GroupPattern[T , F <: T](jGroupPattern: JGroupPattern[T, F])
object GroupPattern {
/**
- * Constructs a new GroupPattern by wrapping a given Java API GroupPattern
- *
- * @param jGroupPattern Underlying Java API GroupPattern.
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- * @return New wrapping GroupPattern object
- */
+ * Constructs a new GroupPattern by wrapping a given Java API GroupPattern
+ *
+ * @param jGroupPattern
+ * Underlying Java API GroupPattern.
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ * @return
+ * New wrapping GroupPattern object
+ */
def apply[T, F <: T](jGroupPattern: JGroupPattern[T, F]) = new GroupPattern[T, F](jGroupPattern)
}
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
index c2d2788c0d8..167603d0823 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
@@ -19,57 +19,49 @@ package org.apache.flink.cep.scala.pattern
import org.apache.flink.cep
import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy
-import org.apache.flink.cep.pattern.conditions.IterativeCondition.{Context => JContext}
+import org.apache.flink.cep.pattern.{MalformedPatternException, Pattern => JPattern, Quantifier}
import org.apache.flink.cep.pattern.conditions.{IterativeCondition, SimpleCondition}
-import org.apache.flink.cep.pattern.{MalformedPatternException, Quantifier, Pattern => JPattern}
+import org.apache.flink.cep.pattern.conditions.IterativeCondition.{Context => JContext}
import org.apache.flink.cep.scala.conditions.Context
import org.apache.flink.streaming.api.windowing.time.Time
/**
- * Base class for a pattern definition.
- *
- * A pattern definition is used by [[org.apache.flink.cep.nfa.compiler.NFACompiler]] to create
- * a [[org.apache.flink.cep.nfa.NFA]].
- *
- * {{{
- * Pattern<T, F> pattern = Pattern.<T>begin("start")
- * .next("middle").subtype(F.class)
- * .followedBy("end").where(new MyCondition());
- * }}}
- *
- * @param jPattern Underlying Java API Pattern
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- */
-class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
+ * Base class for a pattern definition.
+ *
+ * A pattern definition is used by [[org.apache.flink.cep.nfa.compiler.NFACompiler]] to create a
+ * [[org.apache.flink.cep.nfa.NFA]].
+ *
+ * {{{
+ * Pattern<T, F> pattern = Pattern.<T>begin("start")
+ * .next("middle").subtype(F.class)
+ * .followedBy("end").where(new MyCondition());
+ * }}}
+ *
+ * @param jPattern
+ * Underlying Java API Pattern
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ */
+class Pattern[T, F <: T](jPattern: JPattern[T, F]) {
private[flink] def wrappedPattern = jPattern
- /**
- * @return The previous pattern
- */
+ /** @return The previous pattern */
def getPrevious: Option[Pattern[T, _ <: T]] = {
wrapPattern(jPattern.getPrevious)
}
- /**
- *
- * @return Name of the pattern operator
- */
+ /** @return Name of the pattern operator */
def getName: String = jPattern.getName
- /**
- *
- * @return Window length in which the pattern match has to occur
- */
+ /** @return Window length in which the pattern match has to occur */
def getWindowTime: Option[Time] = {
Option(jPattern.getWindowTime)
}
- /**
- *
- * @return currently applied quantifier to this pattern
- */
+ /** @return currently applied quantifier to this pattern */
def getQuantifier: Quantifier = jPattern.getQuantifier
def getCondition: Option[IterativeCondition[F]] = {
@@ -81,30 +73,30 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{AND}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition as an [[IterativeCondition]].
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{AND}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition as an [[IterativeCondition]].
+ * @return
+ * The pattern with the new condition is set.
+ */
def where(condition: IterativeCondition[F]): Pattern[T, F] = {
jPattern.where(condition)
this
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{AND}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition to be set.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{AND}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition to be set.
+ * @return
+ * The pattern with the new condition is set.
+ */
def where(condition: (F, Context[F]) => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -117,15 +109,15 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{AND}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition to be set.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{AND}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition to be set.
+ * @return
+ * The pattern with the new condition is set.
+ */
def where(condition: F => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -136,30 +128,30 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{OR}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The condition as an [[IterativeCondition]].
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{OR}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The condition as an [[IterativeCondition]].
+ * @return
+ * The pattern with the new condition is set.
+ */
def or(condition: IterativeCondition[F]): Pattern[T, F] = {
jPattern.or(condition)
this
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{OR}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The {{{OR}}} condition.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{OR}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The {{{OR}}} condition.
+ * @return
+ * The pattern with the new condition is set.
+ */
def or(condition: F => Boolean): Pattern[T, F] = {
val condFun = new SimpleCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -171,15 +163,15 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Adds a condition that has to be satisfied by an event
- * in order to be considered a match. If another condition has already been
- * set, the new one is going to be combined with the previous with a
- * logical {{{OR}}}. In other case, this is going to be the only
- * condition.
- *
- * @param condition The {{{OR}}} condition.
- * @return The pattern with the new condition is set.
- */
+ * Adds a condition that has to be satisfied by an event in order to be considered a match. If
+ * another condition has already been set, the new one is going to be combined with the previous
+ * with a logical {{{OR}}}. In other case, this is going to be the only condition.
+ *
+ * @param condition
+ * The {{{OR}}} condition.
+ * @return
+ * The pattern with the new condition is set.
+ */
def or(condition: (F, Context[F]) => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(condition)
@@ -191,37 +183,42 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Applies a subtype constraint on the current pattern. This means that an event has
- * to be of the given subtype in order to be matched.
- *
- * @param clazz Class of the subtype
- * @tparam S Type of the subtype
- * @return The same pattern with the new subtype constraint
- */
+ * Applies a subtype constraint on the current pattern. This means that an event has to be of the
+ * given subtype in order to be matched.
+ *
+ * @param clazz
+ * Class of the subtype
+ * @tparam S
+ * Type of the subtype
+ * @return
+ * The same pattern with the new subtype constraint
+ */
def subtype[S <: F](clazz: Class[S]): Pattern[T, S] = {
jPattern.subtype(clazz)
this.asInstanceOf[Pattern[T, S]]
}
/**
- * Applies a stop condition for a looping state. It allows cleaning the underlying state.
- *
- * @param untilCondition a condition an event has to satisfy to stop collecting events into
- * looping state
- * @return The same pattern with applied untilCondition
- */
+ * Applies a stop condition for a looping state. It allows cleaning the underlying state.
+ *
+ * @param untilCondition
+ * a condition an event has to satisfy to stop collecting events into looping state
+ * @return
+ * The same pattern with applied untilCondition
+ */
def until(untilCondition: IterativeCondition[F]): Pattern[T, F] = {
jPattern.until(untilCondition)
this
}
/**
- * Applies a stop condition for a looping state. It allows cleaning the underlying state.
- *
- * @param untilCondition a condition an event has to satisfy to stop collecting events into
- * looping state
- * @return The same pattern with applied untilCondition
- */
+ * Applies a stop condition for a looping state. It allows cleaning the underlying state.
+ *
+ * @param untilCondition
+ * a condition an event has to satisfy to stop collecting events into looping state
+ * @return
+ * The same pattern with applied untilCondition
+ */
def until(untilCondition: (F, Context[F]) => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(untilCondition)
@@ -233,12 +230,13 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Applies a stop condition for a looping state. It allows cleaning the underlying state.
- *
- * @param untilCondition a condition an event has to satisfy to stop collecting events into
- * looping state
- * @return The same pattern with applied untilCondition
- */
+ * Applies a stop condition for a looping state. It allows cleaning the underlying state.
+ *
+ * @param untilCondition
+ * a condition an event has to satisfy to stop collecting events into looping state
+ * @return
+ * The same pattern with applied untilCondition
+ */
def until(untilCondition: F => Boolean): Pattern[T, F] = {
val condFun = new IterativeCondition[F] {
val cleanCond = cep.scala.cleanClosure(untilCondition)
@@ -249,240 +247,270 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
}
/**
- * Defines the maximum time interval in which a matching pattern has to be completed in
- * order to be considered valid. This interval corresponds to the maximum time gap between first
- * and the last event.
- *
- * @param windowTime Time of the matching window
- * @return The same pattern operator with the new window length
- */
+ * Defines the maximum time interval in which a matching pattern has to be completed in order to
+ * be considered valid. This interval corresponds to the maximum time gap between first and the
+ * last event.
+ *
+ * @param windowTime
+ * Time of the matching window
+ * @return
+ * The same pattern operator with the new window length
+ */
def within(windowTime: Time): Pattern[T, F] = {
jPattern.within(windowTime)
this
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces strict
- * temporal contiguity. This means that the whole pattern sequence matches only
- * if an event which matches this pattern directly follows the preceding matching
- * event. Thus, there cannot be any events in between two matching events.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces strict temporal contiguity.
+ * This means that the whole pattern sequence matches only if an event which matches this pattern
+ * directly follows the preceding matching event. Thus, there cannot be any events in between two
+ * matching events.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def next(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.next(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces that there is no event
- * matching this pattern right after the preceding matched event.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces that there is no event
+ * matching this pattern right after the preceding matched event.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def notNext(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.notNext(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedBy(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.followedBy(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces that there is no event
- * matching this pattern between the preceding pattern and succeeding this one.
- *
- * NOTE: There has to be other pattern after this one.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
- def notFollowedBy(name : String): Pattern[T, T] = {
+ * Appends a new pattern to the existing one. The new pattern enforces that there is no event
+ * matching this pattern between the preceding pattern and succeeding this one.
+ *
+ * NOTE: There has to be other pattern after this one.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
+ def notFollowedBy(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.notFollowedBy(name))
}
/**
- * Appends a new pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param name Name of the new pattern
- * @return A new pattern which is appended to this one
- */
+ * Appends a new pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param name
+ * Name of the new pattern
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedByAny(name: String): Pattern[T, T] = {
Pattern[T, T](jPattern.followedByAny(name))
}
-
/**
- * Specifies that this pattern is optional for a final match of the pattern
- * sequence to happen.
- *
- * @return The same pattern as optional.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern is optional for a final match of the pattern sequence to happen.
+ *
+ * @return
+ * The same pattern as optional.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def optional: Pattern[T, F] = {
jPattern.optional()
this
}
/**
- * Specifies that this pattern can occur {{{one or more}}} times.
- * This means at least one and at most infinite number of events can
- * be matched to this pattern.
- *
- * If this quantifier is enabled for a
- * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
- * {{{A1 A2 B}}} appears, this will generate patterns:
- * {{{A1 B}}} and {{{A1 A2 B}}}. See also {{{allowCombinations()}}}.
- *
- * @return The same pattern with a [[Quantifier.looping()]] quantifier applied.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern can occur {{{one or more}}} times. This means at least one and at
+ * most infinite number of events can be matched to this pattern.
+ *
+ * If this quantifier is enabled for a pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of
+ * events {{{A1 A2 B}}} appears, this will generate patterns: {{{A1 B}}} and {{{A1 A2 B}}}. See
+ * also {{{allowCombinations()}}}.
+ *
+ * @return
+ * The same pattern with a [[Quantifier.looping()]] quantifier applied.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def oneOrMore: Pattern[T, F] = {
jPattern.oneOrMore()
this
}
/**
- * Specifies that this pattern is greedy.
- * This means as many events as possible will be matched to this pattern.
- *
- * @return The same pattern with { @link Quantifier#greedy} set to true.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern is greedy. This means as many events as possible will be matched to
+ * this pattern.
+ *
+ * @return
+ * The same pattern with { @link Quantifier#greedy} set to true.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def greedy: Pattern[T, F] = {
jPattern.greedy()
this
}
/**
- * Specifies exact number of times that this pattern should be matched.
- *
- * @param times number of times matching event must appear
- * @return The same pattern with number of times applied
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies exact number of times that this pattern should be matched.
+ *
+ * @param times
+ * number of times matching event must appear
+ * @return
+ * The same pattern with number of times applied
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def times(times: Int): Pattern[T, F] = {
jPattern.times(times)
this
}
/**
- * Specifies that the pattern can occur between from and to times.
- *
- * @param from number of times matching event must appear at least
- * @param to number of times matching event must appear at most
- * @return The same pattern with the number of times range applied
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that the pattern can occur between from and to times.
+ *
+ * @param from
+ * number of times matching event must appear at least
+ * @param to
+ * number of times matching event must appear at most
+ * @return
+ * The same pattern with the number of times range applied
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def times(from: Int, to: Int): Pattern[T, F] = {
jPattern.times(from, to)
this
}
/**
- * Specifies that this pattern can occur the specified times at least.
- * This means at least the specified times and at most infinite number of events can
- * be matched to this pattern.
- *
- * @return The same pattern with a { @link Quantifier#looping(ConsumingStrategy)} quantifier
- * applied.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Specifies that this pattern can occur the specified times at least. This means at least the
+ * specified times and at most infinite number of events can be matched to this pattern.
+ *
+ * @return
+ * The same pattern with a { @link Quantifier#looping(ConsumingStrategy)} quantifier applied.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def timesOrMore(times: Int): Pattern[T, F] = {
jPattern.timesOrMore(times)
this
}
/**
- * Applicable only to [[Quantifier.looping()]] and [[Quantifier.times()]] patterns,
- * this option allows more flexibility to the matching events.
- *
- * If {{{allowCombinations()}}} is not applied for a
- * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
- * {{{A1 A2 B}}} appears, this will generate patterns:
- * {{{A1 B}}} and {{{A1 A2 B}}}. If this method is applied, we
- * will have {{{A1 B}}}, {{{A2 B}}} and {{{A1 A2 B}}}.
- *
- * @return The same pattern with the updated quantifier.
- * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
- */
+ * Applicable only to [[Quantifier.looping()]] and [[Quantifier.times()]] patterns, this option
+ * allows more flexibility to the matching events.
+ *
+ * If {{{allowCombinations()}}} is not applied for a pattern {{{A.oneOrMore().followedBy(B)}}} and
+ * a sequence of events {{{A1 A2 B}}} appears, this will generate patterns: {{{A1 B}}} and
+ * {{{A1 A2 B}}}. If this method is applied, we will have {{{A1 B}}}, {{{A2 B}}} and
+ * {{{A1 A2 B}}}.
+ *
+ * @return
+ * The same pattern with the updated quantifier.
+ * @throws MalformedPatternException
+ * if the quantifier is not applicable to this pattern.
+ */
def allowCombinations(): Pattern[T, F] = {
jPattern.allowCombinations()
this
}
/**
- * Works in conjunction with [[Pattern#oneOrMore()]] or [[Pattern#times(int)]].
- * Specifies that any not matching element breaks the loop.
- *
- * E.g. a pattern like:
- * {{{
- * Pattern.begin("start").where(_.getName().equals("c"))
- * .followedBy("middle").where(_.getName().equals("a")).oneOrMore().consecutive()
- * .followedBy("end1").where(_.getName().equals("b"));
- * }}}
- *
- * For a sequence: C D A1 A2 A3 D A4 B
- *
- * will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
- *
- * By default a relaxed continuity is applied.
- * @return pattern with continuity changed to strict
- */
+ * Works in conjunction with [[Pattern#oneOrMore()]] or [[Pattern#times(int)]]. Specifies that any
+ * not matching element breaks the loop.
+ *
+ * E.g. a pattern like:
+ * {{{
+ * Pattern.begin("start").where(_.getName().equals("c"))
+ * .followedBy("middle").where(_.getName().equals("a")).oneOrMore().consecutive()
+ * .followedBy("end1").where(_.getName().equals("b"));
+ * }}}
+ *
+ * For a sequence: C D A1 A2 A3 D A4 B
+ *
+ * will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
+ *
+ * By default a relaxed continuity is applied.
+ * @return
+ * pattern with continuity changed to strict
+ */
def consecutive(): Pattern[T, F] = {
jPattern.consecutive()
this
}
/**
- * Appends a new group pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param pattern the pattern to append
- * @return A new pattern which is appended to this one
- */
+ * Appends a new group pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param pattern
+ * the pattern to append
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedBy(pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](jPattern.followedBy(pattern.wrappedPattern))
/**
- * Appends a new group pattern to the existing one. The new pattern enforces non-strict
- * temporal contiguity. This means that a matching event of this pattern and the
- * preceding matching event might be interleaved with other events which are ignored.
- *
- * @param pattern the pattern to append
- * @return A new pattern which is appended to this one
- */
+ * Appends a new group pattern to the existing one. The new pattern enforces non-strict temporal
+ * contiguity. This means that a matching event of this pattern and the preceding matching event
+ * might be interleaved with other events which are ignored.
+ *
+ * @param pattern
+ * the pattern to append
+ * @return
+ * A new pattern which is appended to this one
+ */
def followedByAny(pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](jPattern.followedByAny(pattern.wrappedPattern))
/**
- * Appends a new group pattern to the existing one. The new pattern enforces strict
- * temporal contiguity. This means that the whole pattern sequence matches only
- * if an event which matches this pattern directly follows the preceding matching
- * event. Thus, there cannot be any events in between two matching events.
- *
- * @param pattern the pattern to append
- * @return A new pattern which is appended to this one
- */
+ * Appends a new group pattern to the existing one. The new pattern enforces strict temporal
+ * contiguity. This means that the whole pattern sequence matches only if an event which matches
+ * this pattern directly follows the preceding matching event. Thus, there cannot be any events in
+ * between two matching events.
+ *
+ * @param pattern
+ * the pattern to append
+ * @return
+ * A new pattern which is appended to this one
+ */
def next(pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](jPattern.next(pattern.wrappedPattern))
/**
- * Get after match skip strategy.
- * @return current after match skip strategy
- */
+ * Get after match skip strategy.
+ * @return
+ * current after match skip strategy
+ */
def getAfterMatchSkipStrategy: AfterMatchSkipStrategy =
jPattern.getAfterMatchSkipStrategy
}
@@ -490,56 +518,71 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
object Pattern {
/**
- * Constructs a new Pattern by wrapping a given Java API Pattern
- *
- * @param jPattern Underlying Java API Pattern.
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- * @return New wrapping Pattern object
- */
+ * Constructs a new Pattern by wrapping a given Java API Pattern
+ *
+ * @param jPattern
+ * Underlying Java API Pattern.
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ * @return
+ * New wrapping Pattern object
+ */
def apply[T, F <: T](jPattern: JPattern[T, F]) = new Pattern[T, F](jPattern)
/**
- * Starts a new pattern sequence. The provided name is the one of the initial pattern
- * of the new sequence. Furthermore, the base type of the event sequence is set.
- *
- * @param name The name of starting pattern of the new pattern sequence
- * @tparam X Base type of the event pattern
- * @return The first pattern of a pattern sequence
- */
+ * Starts a new pattern sequence. The provided name is the one of the initial pattern of the new
+ * sequence. Furthermore, the base type of the event sequence is set.
+ *
+ * @param name
+ * The name of starting pattern of the new pattern sequence
+ * @tparam X
+ * Base type of the event pattern
+ * @return
+ * The first pattern of a pattern sequence
+ */
def begin[X](name: String): Pattern[X, X] = Pattern(JPattern.begin(name))
/**
- * Starts a new pattern sequence. The provided name is the one of the initial pattern
- * of the new sequence. Furthermore, the base type of the event sequence is set.
- *
- * @param name The name of starting pattern of the new pattern sequence
- * @param afterMatchSkipStrategy The skip strategy to use after each match
- * @tparam X Base type of the event pattern
- * @return The first pattern of a pattern sequence
- */
+ * Starts a new pattern sequence. The provided name is the one of the initial pattern of the new
+ * sequence. Furthermore, the base type of the event sequence is set.
+ *
+ * @param name
+ * The name of starting pattern of the new pattern sequence
+ * @param afterMatchSkipStrategy
+ * The skip strategy to use after each match
+ * @tparam X
+ * Base type of the event pattern
+ * @return
+ * The first pattern of a pattern sequence
+ */
def begin[X](name: String, afterMatchSkipStrategy: AfterMatchSkipStrategy): Pattern[X, X] =
Pattern(JPattern.begin(name, afterMatchSkipStrategy))
/**
- * Starts a new pattern sequence. The provided pattern is the initial pattern
- * of the new sequence.
- *
- * @param pattern the pattern to begin with
- * @return the first pattern of a pattern sequence
- */
+ * Starts a new pattern sequence. The provided pattern is the initial pattern of the new sequence.
+ *
+ * @param pattern
+ * the pattern to begin with
+ * @return
+ * the first pattern of a pattern sequence
+ */
def begin[T, F <: T](pattern: Pattern[T, F]): GroupPattern[T, F] =
GroupPattern[T, F](JPattern.begin(pattern.wrappedPattern))
/**
- * Starts a new pattern sequence. The provided pattern is the initial pattern
- * of the new sequence.
- *
- * @param pattern the pattern to begin with
- * @param afterMatchSkipStrategy The skip strategy to use after each match
- * @return The first pattern of a pattern sequence
- */
- def begin[T, F <: T](pattern: Pattern[T, F],
+ * Starts a new pattern sequence. The provided pattern is the initial pattern of the new sequence.
+ *
+ * @param pattern
+ * the pattern to begin with
+ * @param afterMatchSkipStrategy
+ * The skip strategy to use after each match
+ * @return
+ * The first pattern of a pattern sequence
+ */
+ def begin[T, F <: T](
+ pattern: Pattern[T, F],
afterMatchSkipStrategy: AfterMatchSkipStrategy): GroupPattern[T, F] =
GroupPattern(JPattern.begin(pattern.wrappedPattern, afterMatchSkipStrategy))
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
index 13305dd0413..0c3dab9b88a 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
@@ -20,29 +20,35 @@ package org.apache.flink.cep.scala
import org.apache.flink.cep.pattern.{Pattern => JPattern}
import org.apache.flink.cep.pattern.conditions.IterativeCondition.{Context => JContext}
import org.apache.flink.cep.scala.conditions.Context
+
import scala.collection.JavaConverters._
package object pattern {
+
/**
- * Utility method to wrap [[org.apache.flink.cep.pattern.Pattern]] and its subclasses
- * for usage with the Scala API.
- *
- * @param javaPattern The underlying pattern from the Java API
- * @tparam T Base type of the elements appearing in the pattern
- * @tparam F Subtype of T to which the current pattern operator is constrained
- * @return A pattern from the Scala API which wraps the pattern from the Java API
- */
- private[flink] def wrapPattern[T, F <: T](javaPattern: JPattern[T, F])
- : Option[Pattern[T, F]] = javaPattern match {
- case p: JPattern[T, F] => Some(Pattern[T, F](p))
- case _ => None
- }
+ * Utility method to wrap [[org.apache.flink.cep.pattern.Pattern]] and its subclasses for usage
+ * with the Scala API.
+ *
+ * @param javaPattern
+ * The underlying pattern from the Java API
+ * @tparam T
+ * Base type of the elements appearing in the pattern
+ * @tparam F
+ * Subtype of T to which the current pattern operator is constrained
+ * @return
+ * A pattern from the Scala API which wraps the pattern from the Java API
+ */
+ private[flink] def wrapPattern[T, F <: T](javaPattern: JPattern[T, F]): Option[Pattern[T, F]] =
+ javaPattern match {
+ case p: JPattern[T, F] => Some(Pattern[T, F](p))
+ case _ => None
+ }
private[pattern] class JContextWrapper[F](private val jContext: JContext[F])
- extends Context[F] with Serializable {
+ extends Context[F]
+ with Serializable {
override def getEventsForPattern(name: String): Iterable[F] =
jContext.getEventsForPattern(name).asScala
}
}
-
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala
index 560d99af4a6..01fab9908d3 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/CEPScalaAPICompletenessTest.scala
@@ -17,17 +17,18 @@
*/
package org.apache.flink.cep.scala
-import java.lang.reflect.Method
-
import org.apache.flink.api.scala.completeness.ScalaAPICompletenessTestBase
import org.apache.flink.cep.{PatternStream => JPatternStream}
+
import org.junit.Test
+import java.lang.reflect.Method
+
import scala.language.existentials
/**
- * This checks whether the CEP Scala API is up to feature parity with the Java API.
- * Implements the [[ScalaAPICompletenessTestBase]] for CEP.
+ * This checks whether the CEP Scala API is up to feature parity with the Java API. Implements the
+ * [[ScalaAPICompletenessTestBase]] for CEP.
*/
class CEPScalaAPICompletenessTest extends ScalaAPICompletenessTestBase {
@@ -39,7 +40,9 @@ class CEPScalaAPICompletenessTest extends ScalaAPICompletenessTestBase {
@Test
override def testCompleteness(): Unit = {
- checkMethods("PatternStream", "PatternStream",
+ checkMethods(
+ "PatternStream",
+ "PatternStream",
classOf[JPatternStream[_]],
classOf[PatternStream[_]])
}
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala
index 80d96aca175..a8f1921bce7 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala
@@ -17,8 +17,6 @@
*/
package org.apache.flink.cep.scala
-import java.lang
-
import org.apache.flink.api.common.functions.RuntimeContext
import org.apache.flink.api.common.functions.util.{FunctionUtils, ListCollector}
import org.apache.flink.cep.functions.{PatternProcessFunction, TimedOutPartialMatchHandler}
@@ -29,13 +27,16 @@ import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.transformations.OneInputTransformation
import org.apache.flink.util
import org.apache.flink.util.{Collector, TestLogger}
+
import org.junit.Assert._
import org.junit.Test
import org.mockito.Mockito
+import java.lang
+
+import scala.collection.{mutable, Map}
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
-import scala.collection.{Map, mutable}
class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
@@ -48,18 +49,19 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
val pStream: PatternStream[(Int, Int)] = CEP.pattern(dummyDataStream, pattern)
val param = Map("begin" -> List((1, 2)))
val result: DataStream[(Int, Int)] = pStream
- .select((pattern: Map[String, Iterable[(Int, Int)]]) => {
- //verifies input parameter forwarding
- assertEquals(param, pattern)
- param("begin").head
- })
+ .select(
+ (pattern: Map[String, Iterable[(Int, Int)]]) => {
+ // verifies input parameter forwarding
+ assertEquals(param, pattern)
+ param("begin").head
+ })
val outList = new java.util.ArrayList[(Int, Int)]
val outParam = new ListCollector[(Int, Int)](outList)
val fun = extractFun[(Int, Int), (Int, Int)](result)
fun.processMatch(param.mapValues(_.asJava).asJava, new ListTestContext, outParam)
- //verifies output parameter forwarding
+ // verifies output parameter forwarding
assertEquals(param("begin").head, outList.get(0))
}
@@ -76,16 +78,16 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
val outParam = new ListCollector[List[Int]](outList)
val result: DataStream[List[Int]] = pStream
-
- .flatSelect((pattern: Map[String, Iterable[List[Int]]], out: Collector[List[Int]]) => {
- //verifies input parameter forwarding
- assertEquals(inParam, pattern)
- out.collect(pattern("begin").head)
- })
+ .flatSelect(
+ (pattern: Map[String, Iterable[List[Int]]], out: Collector[List[Int]]) => {
+ // verifies input parameter forwarding
+ assertEquals(inParam, pattern)
+ out.collect(pattern("begin").head)
+ })
val fun = extractFun[List[Int], List[Int]](result)
fun.processMatch(inParam.mapValues(_.asJava).asJava, new ListTestContext, outParam)
- //verify output parameter forwarding and that flatMap function was actually called
+ // verify output parameter forwarding and that flatMap function was actually called
assertEquals(inList, outList.get(0))
}
@@ -102,23 +104,26 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
val outputTag = OutputTag[Either[String, String]]("timeouted")
val result: DataStream[Either[String, String]] = pStream.flatSelect(outputTag) {
- (pattern: Map[String, Iterable[String]], timestamp: Long,
- out: Collector[Either[String, String]]) =>
- out.collect(Left("timeout"))
- out.collect(Left(pattern("begin").head))
- } {
- (pattern: Map[String, Iterable[String]], out: Collector[Either[String, String]]) =>
- //verifies input parameter forwarding
- assertEquals(inParam, pattern)
- out.collect(Right("match"))
- out.collect(Right(pattern("begin").head))
- }
+ (
+ pattern: Map[String, Iterable[String]],
+ timestamp: Long,
+ out: Collector[Either[String, String]]) =>
+ out.collect(Left("timeout"))
+ out.collect(Left(pattern("begin").head))
+ } {
+ (pattern: Map[String, Iterable[String]], out: Collector[Either[String, String]]) =>
+ // verifies input parameter forwarding
+ assertEquals(inParam, pattern)
+ out.collect(Right("match"))
+ out.collect(Right(pattern("begin").head))
+ }
val fun = extractFun[String, Either[String, String]](result)
val ctx = new ListTestContext
fun.processMatch(inParam.mapValues(_.asJava).asJava, ctx, output)
- fun.asInstanceOf[TimedOutPartialMatchHandler[String]]
+ fun
+ .asInstanceOf[TimedOutPartialMatchHandler[String]]
.processTimedOutMatch(inParam.mapValues(_.asJava).asJava, ctx)
assertEquals(List(Right("match"), Right("barfoo")).asJava, outList)
@@ -126,8 +131,7 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger {
}
def extractFun[IN, OUT](dataStream: DataStream[OUT]): PatternProcessFunction[IN, OUT] = {
- val oper = dataStream.javaStream
- .getTransformation
+ val oper = dataStream.javaStream.getTransformation
.asInstanceOf[OneInputTransformation[_, _]]
.getOperator
.asInstanceOf[CepOperator[IN, Byte, OUT]]
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala
index ac72bdffb3b..941553eadc7 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternScalaAPICompletenessTest.scala
@@ -17,17 +17,18 @@
*/
package org.apache.flink.cep.scala.pattern
-import java.lang.reflect.Method
-
import org.apache.flink.api.scala.completeness.ScalaAPICompletenessTestBase
import org.apache.flink.cep.pattern.{Pattern => JPattern}
+
import org.junit.Test
+import java.lang.reflect.Method
+
import scala.language.existentials
/**
- * This checks whether the CEP Scala API is up to feature parity with the Java API.
- * Implements the [[ScalaAPICompletenessTestBase]] for CEP.
+ * This checks whether the CEP Scala API is up to feature parity with the Java API. Implements the
+ * [[ScalaAPICompletenessTestBase]] for CEP.
*/
class PatternScalaAPICompletenessTest extends ScalaAPICompletenessTestBase {
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
index 91a15032a91..23453c5d167 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
@@ -17,28 +17,28 @@
*/
package org.apache.flink.cep.scala.pattern
-import org.apache.flink.cep.pattern.{Pattern => JPattern}
-import org.junit.Assert._
-import org.junit.Test
import org.apache.flink.cep.Event
import org.apache.flink.cep.SubEvent
+import org.apache.flink.cep.pattern.{Pattern => JPattern}
import org.apache.flink.cep.pattern.Quantifier.ConsumingStrategy
import org.apache.flink.cep.pattern.conditions._
import org.apache.flink.cep.scala.conditions.Context
+import org.junit.Assert._
+import org.junit.Test
+
class PatternTest {
/**
- * These tests simply check that the pattern construction completes without failure and that the
- * Scala API pattern is synchronous with its wrapped Java API pattern.
- */
+ * These tests simply check that the pattern construction completes without failure and that the
+ * Scala API pattern is synchronous with its wrapped Java API pattern.
+ */
@Test
def testStrictContiguity(): Unit = {
val pattern = Pattern.begin[Event]("start").next("next").next("end")
val jPattern = JPattern.begin[Event]("start").next("next").next("end")
-
assertTrue(checkCongruentRepresentations(pattern, jPattern))
assertTrue(checkCongruentRepresentations(wrapPattern(jPattern).get, jPattern))
@@ -55,7 +55,6 @@ class PatternTest {
assertEquals(preprevious.getName, "start")
}
-
@Test
def testNonStrictContiguity(): Unit = {
val pattern = Pattern.begin[Event]("start").followedBy("next").followedBy("end")
@@ -80,20 +79,23 @@ class PatternTest {
@Test
def testStrictContiguityWithCondition(): Unit = {
- val pattern = Pattern.begin[Event]("start")
+ val pattern = Pattern
+ .begin[Event]("start")
.next("next")
.where((value: Event, _: Context[Event]) => value.getName == "foobar")
.next("end")
.where((value: Event, _: Context[Event]) => value.getId == 42)
- val jPattern = JPattern.begin[Event]("start")
+ val jPattern = JPattern
+ .begin[Event]("start")
.next("next")
.where(new SimpleCondition[Event]() {
@throws[Exception]
def filter(value: Event): Boolean = {
value.getName == "foobar"
}
- }).next("end")
+ })
+ .next("end")
.where(new SimpleCondition[Event]() {
@throws[Exception]
def filter(value: Event): Boolean = {
@@ -122,12 +124,14 @@ class PatternTest {
@Test
def testPatternWithSubtyping(): Unit = {
- val pattern = Pattern.begin[Event]("start")
+ val pattern = Pattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.followedBy("end")
- val jPattern = JPattern.begin[Event]("start")
+ val jPattern = JPattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.followedBy("end")
@@ -152,13 +156,15 @@ class PatternTest {
@Test
def testPatternWithSubtypingAndFilter(): Unit = {
- val pattern = Pattern.begin[Event]("start")
+ val pattern = Pattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.where(_ => false)
.followedBy("end")
- val jpattern = JPattern.begin[Event]("start")
+ val jpattern = JPattern
+ .begin[Event]("start")
.next("subevent")
.subtype(classOf[SubEvent])
.where(new SimpleCondition[SubEvent]() {
@@ -166,12 +172,12 @@ class PatternTest {
def filter(value: SubEvent): Boolean = {
false
}
- }).followedBy("end")
+ })
+ .followedBy("end")
assertTrue(checkCongruentRepresentations(pattern, jpattern))
assertTrue(checkCongruentRepresentations(wrapPattern(jpattern).get, jpattern))
-
val previous = pattern.getPrevious.orNull
val preprevious = previous.getPrevious.orNull
@@ -187,61 +193,58 @@ class PatternTest {
assertEquals(preprevious.getName, "start")
}
- def checkCongruentRepresentations[T, _ <: T](pattern: Pattern[T, _ <: T],
- jPattern: JPattern[T, _ <: T]): Boolean = {
+ def checkCongruentRepresentations[T, _ <: T](
+ pattern: Pattern[T, _ <: T],
+ jPattern: JPattern[T, _ <: T]): Boolean = {
((pattern == null && jPattern == null)
- || (pattern != null && jPattern != null)
- //check equal pattern names
- && threeWayEquals(
- pattern.getName,
- pattern.wrappedPattern.getName,
- jPattern.getName)
- //check equal time windows
- && threeWayEquals(
+ || (pattern != null && jPattern != null)
+ // check equal pattern names
+ && threeWayEquals(pattern.getName, pattern.wrappedPattern.getName, jPattern.getName)
+ // check equal time windows
+ && threeWayEquals(
pattern.getWindowTime.orNull,
pattern.wrappedPattern.getWindowTime,
jPattern.getWindowTime)
- //check congruent class names / types
- && threeWayEquals(
+ // check congruent class names / types
+ && threeWayEquals(
pattern.getClass.getSimpleName,
pattern.wrappedPattern.getClass.getSimpleName,
jPattern.getClass.getSimpleName)
- //best effort to confirm congruent filter functions
- && compareFilterFunctions(
- pattern.getCondition.orNull,
- jPattern.getCondition)
- //recursively check previous patterns
- && checkCongruentRepresentations(
- pattern.getPrevious.orNull,
- jPattern.getPrevious))
+ // best effort to confirm congruent filter functions
+ && compareFilterFunctions(pattern.getCondition.orNull, jPattern.getCondition)
+ // recursively check previous patterns
+ && checkCongruentRepresentations(pattern.getPrevious.orNull, jPattern.getPrevious))
}
def threeWayEquals(a: AnyRef, b: AnyRef, c: AnyRef): Boolean = {
a == b && b == c
}
- def compareFilterFunctions(sFilter: IterativeCondition[_],
- jFilter: IterativeCondition[_]): Boolean = {
+ def compareFilterFunctions(
+ sFilter: IterativeCondition[_],
+ jFilter: IterativeCondition[_]): Boolean = {
+
/**
- * We would like to simply compare the filter functions like this:
- *
- * {{{(pattern.getFilterFunction.orNull == jPattern.getFilterFunction)}}}
- *
- * However, the closure cleaning makes comparing filter functions by reference impossible.
- * Testing for functional equivalence is an undecidable problem. Thus, for do a best effort by
- * simply matching the presence/absence and known classes of filter functions in the patterns.
- */
+ * We would like to simply compare the filter functions like this:
+ *
+ * {{{(pattern.getFilterFunction.orNull == jPattern.getFilterFunction)}}}
+ *
+ * However, the closure cleaning makes comparing filter functions by reference impossible.
+ * Testing for functional equivalence is an undecidable problem. Thus, for do a best effort by
+ * simply matching the presence/absence and known classes of filter functions in the patterns.
+ */
(sFilter, jFilter) match {
- //matching types: and-filter; branch and recurse for inner filters
- case (saf: AndCondition[_], jaf: AndCondition[_])
- => (compareFilterFunctions(saf.getLeft, jaf.getLeft)
- && compareFilterFunctions(saf.getRight, jaf.getRight))
- //matching types: subtype-filter
+ // matching types: and-filter; branch and recurse for inner filters
+ case (saf: AndCondition[_], jaf: AndCondition[_]) => (
+ compareFilterFunctions(saf.getLeft, jaf.getLeft)
+ && compareFilterFunctions(saf.getRight, jaf.getRight)
+ )
+ // matching types: subtype-filter
case (_: SubtypeCondition[_], _: SubtypeCondition[_]) => true
- //mismatch: one-sided and/subtype-filter
+ // mismatch: one-sided and/subtype-filter
case (_: AndCondition[_] | _: SubtypeCondition[_], _) => false
case (_, _: AndCondition[_] | _: SubtypeCondition[_]) => false
- //from here we can only check mutual presence or absence of a function
+ // from here we can only check mutual presence or absence of a function
case (_: IterativeCondition[_], _: IterativeCondition[_]) => true
case (null, null) => true
case _ => false
diff --git a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala
index d3f2f92199e..d5101c22559 100644
--- a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala
+++ b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/ConnectedComponents.scala
@@ -15,11 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.examples
-import java.lang.Long
-
import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.scala._
import org.apache.flink.graph.Edge
@@ -28,25 +25,21 @@ import org.apache.flink.graph.library.GSAConnectedComponents
import org.apache.flink.graph.scala._
import org.apache.flink.types.NullValue
+import java.lang.Long
+
/**
- * This example shows how to use Gelly's library methods.
- * You can find all available library methods in [[org.apache.flink.graph.library]].
- *
- * In particular, this example uses the
- * [[GSAConnectedComponents]]
- * library method to compute the connected components of the input graph.
+ * This example shows how to use Gelly's library methods. You can find all available library methods
+ * in [[org.apache.flink.graph.library]].
+ *
+ * In particular, this example uses the [[GSAConnectedComponents]] library method to compute the
+ * connected components of the input graph.
*
- * The input file is a plain text file and must be formatted as follows:
- * Edges are represented by tuples of srcVertexId, trgVertexId which are
- * separated by tabs. Edges themselves are separated by newlines.
- * For example: <code>1\t2\n1\t3\n</code> defines two edges,
- * 1-2 and 1-3.
+ * The input file is a plain text file and must be formatted as follows: Edges are represented by
+ * tuples of srcVertexId, trgVertexId which are separated by tabs. Edges themselves are separated by
+ * newlines. For example: <code>1\t2\n1\t3\n</code> defines two edges, 1-2 and 1-3.
*
- * Usage {{
- * ConnectedComponents <edge path> <result path> <number of iterations>
- * }}
- * If no parameters are provided, the program is run with default data from
- * [[ConnectedComponentsDefaultData]]
+ * Usage {{ ConnectedComponents <edge path> <result path> <number of iterations> }} If no parameters
+ * are provided, the program is run with default data from [[ConnectedComponentsDefaultData]]
*/
object ConnectedComponents {
def main(args: Array[String]) {
@@ -60,7 +53,6 @@ object ConnectedComponents {
val components = graph.run(new GSAConnectedComponents[Long, Long, NullValue](maxIterations))
-
// emit result
if (fileOutput) {
components.writeAsCsv(outputPath, "\n", ",")
@@ -70,7 +62,7 @@ object ConnectedComponents {
}
}
- private final class InitVertices extends MapFunction[Long, Long] {
+ final private class InitVertices extends MapFunction[Long, Long] {
override def map(id: Long) = id
}
@@ -78,44 +70,47 @@ object ConnectedComponents {
// UTIL METHODS
// ***********************************************************************
- private var fileOutput = false
- private var edgesInputPath: String = null
- private var outputPath: String = null
- private var maxIterations = ConnectedComponentsDefaultData.MAX_ITERATIONS
+ private var fileOutput = false
+ private var edgesInputPath: String = null
+ private var outputPath: String = null
+ private var maxIterations = ConnectedComponentsDefaultData.MAX_ITERATIONS
- private def parseParameters(args: Array[String]): Boolean = {
- if(args.length > 0) {
- if(args.length != 3) {
- System.err.println("Usage ConnectedComponents <edge path> <output path> " +
+ private def parseParameters(args: Array[String]): Boolean = {
+ if (args.length > 0) {
+ if (args.length != 3) {
+ System.err.println(
+ "Usage ConnectedComponents <edge path> <output path> " +
"<num iterations>")
- }
- fileOutput = true
- edgesInputPath = args(0)
- outputPath = args(1)
- maxIterations = 2
- } else {
- System.out.println("Executing ConnectedComponents example with default parameters" +
+ }
+ fileOutput = true
+ edgesInputPath = args(0)
+ outputPath = args(1)
+ maxIterations = 2
+ } else {
+ System.out.println(
+ "Executing ConnectedComponents example with default parameters" +
" and built-in default data.")
- System.out.println(" Provide parameters to read input data from files.")
- System.out.println(" See the documentation for the correct format of input files.")
- System.out.println("Usage ConnectedComponents <edge path> <output path> " +
+ System.out.println(" Provide parameters to read input data from files.")
+ System.out.println(" See the documentation for the correct format of input files.")
+ System.out.println(
+ "Usage ConnectedComponents <edge path> <output path> " +
"<num iterations>")
- }
- true
}
+ true
+ }
- private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, NullValue]] = {
- if (fileOutput) {
- env.readCsvFile[(Long, Long)](edgesInputPath,
- lineDelimiter = "\n",
- fieldDelimiter = "\t")
- .map(edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
- } else {
- val edgeData = ConnectedComponentsDefaultData.DEFAULT_EDGES map {
- case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
- }
- env.fromCollection(edgeData).map(
- edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
+ private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, NullValue]] = {
+ if (fileOutput) {
+ env
+ .readCsvFile[(Long, Long)](edgesInputPath, lineDelimiter = "\n", fieldDelimiter = "\t")
+ .map(edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
+ } else {
+ val edgeData = ConnectedComponentsDefaultData.DEFAULT_EDGES.map {
+ case Array(x, y) => (x.asInstanceOf[Long], y.asInstanceOf[Long])
}
+ env
+ .fromCollection(edgeData)
+ .map(edge => new Edge[Long, NullValue](edge._1, edge._2, NullValue.getInstance))
}
+ }
}
diff --git a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala
index bd5c63bb4f2..672868648b5 100644
--- a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala
+++ b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/GSASingleSourceShortestPaths.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.examples
import org.apache.flink.api.common.functions.MapFunction
@@ -28,14 +27,13 @@ import org.apache.flink.graph.scala.utils.Tuple3ToEdgeMap
/**
* This example shows how to use Gelly's gather-sum-apply iterations.
- *
- * It is an implementation of the Single-Source-Shortest-Paths algorithm.
*
- * The input file is a plain text file and must be formatted as follows:
- * Edges are represented by tuples of srcVertexId, trgVertexId, distance which are
- * separated by tabs. Edges themselves are separated by newlines.
- * For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges,
- * edge 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
+ * It is an implementation of the Single-Source-Shortest-Paths algorithm.
+ *
+ * The input file is a plain text file and must be formatted as follows: Edges are represented by
+ * tuples of srcVertexId, trgVertexId, distance which are separated by tabs. Edges themselves are
+ * separated by newlines. For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges, edge
+ * 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
*
* If no parameters are provided, the program is run with default data from
* [[SingleSourceShortestPathsData]]
@@ -51,8 +49,11 @@ object GSASingleSourceShortestPaths {
val graph = Graph.fromDataSet[Long, Double, Double](edges, new InitVertices(srcVertexId), env)
// Execute the gather-sum-apply iteration
- val result = graph.runGatherSumApplyIteration(new CalculateDistances, new ChooseMinDistance,
- new UpdateDistance, maxIterations)
+ val result = graph.runGatherSumApplyIteration(
+ new CalculateDistances,
+ new ChooseMinDistance,
+ new UpdateDistance,
+ maxIterations)
// Extract the vertices as the result
val singleSourceShortestPaths = result.getVertices
@@ -70,7 +71,7 @@ object GSASingleSourceShortestPaths {
// Single Source Shortest Path UDFs
// --------------------------------------------------------------------------------------------
- private final class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
+ final private class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
override def map(id: Long) = {
if (id.equals(srcId)) {
@@ -81,19 +82,19 @@ object GSASingleSourceShortestPaths {
}
}
- private final class CalculateDistances extends GatherFunction[Double, Double, Double] {
+ final private class CalculateDistances extends GatherFunction[Double, Double, Double] {
override def gather(neighbor: Neighbor[Double, Double]) = {
neighbor.getNeighborValue + neighbor.getEdgeValue
}
}
- private final class ChooseMinDistance extends SumFunction[Double, Double, Double] {
+ final private class ChooseMinDistance extends SumFunction[Double, Double, Double] {
override def sum(newValue: Double, currentValue: Double) = {
Math.min(newValue, currentValue)
}
}
- private final class UpdateDistance extends ApplyFunction[Long, Double, Double] {
+ final private class UpdateDistance extends ApplyFunction[Long, Double, Double] {
override def apply(newDistance: Double, oldDistance: Double) = {
if (newDistance < oldDistance) {
setResult(newDistance)
@@ -112,10 +113,11 @@ object GSASingleSourceShortestPaths {
private var maxIterations = 5
private def parseParameters(args: Array[String]): Boolean = {
- if(args.length > 0) {
- if(args.length != 4) {
- System.err.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ if (args.length > 0) {
+ if (args.length != 4) {
+ System.err.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
fileOutput = true
srcVertexId = args(0).toLong
@@ -123,26 +125,29 @@ object GSASingleSourceShortestPaths {
outputPath = args(2)
maxIterations = 3
} else {
- System.out.println("Executing Single Source Shortest Paths example "
- + "with default parameters and built-in default data.")
+ System.out.println(
+ "Executing Single Source Shortest Paths example "
+ + "with default parameters and built-in default data.")
System.out.println(" Provide parameters to read input data from files.")
System.out.println(" See the documentation for the correct format of input files.")
- System.out.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ System.out.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
true
}
private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, Double]] = {
if (fileOutput) {
- env.readCsvFile[(Long, Long, Double)](edgesInputPath,
- lineDelimiter = "\n",
- fieldDelimiter = "\t")
+ env
+ .readCsvFile[(Long, Long, Double)](
+ edgesInputPath,
+ lineDelimiter = "\n",
+ fieldDelimiter = "\t")
.map(new Tuple3ToEdgeMap[Long, Double]())
} else {
- val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES map {
- case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long],
- z.asInstanceOf[Double])
+ val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES.map {
+ case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long], z.asInstanceOf[Double])
}
env.fromCollection(edgeData).map(new Tuple3ToEdgeMap[Long, Double]())
}
diff --git a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala
index f3b10f7d6f9..85ca764a855 100644
--- a/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala
+++ b/flink-libraries/flink-gelly-examples/src/main/scala/org/apache/flink/graph/scala/examples/SingleSourceShortestPaths.scala
@@ -15,29 +15,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.examples
import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.scala._
+import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.graph.examples.data.SingleSourceShortestPathsData
import org.apache.flink.graph.scala._
import org.apache.flink.graph.scala.utils.Tuple3ToEdgeMap
import org.apache.flink.graph.spargel.{GatherFunction, MessageIterator, ScatterFunction}
-import org.apache.flink.graph.{Edge, Vertex}
import scala.collection.JavaConversions._
/**
* This example shows how to use Gelly's scatter-gather iterations.
- *
- * It is an implementation of the Single-Source-Shortest-Paths algorithm.
*
- * The input file is a plain text file and must be formatted as follows:
- * Edges are represented by tuples of srcVertexId, trgVertexId, distance which are
- * separated by tabs. Edges themselves are separated by newlines.
- * For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges,
- * edge 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
+ * It is an implementation of the Single-Source-Shortest-Paths algorithm.
+ *
+ * The input file is a plain text file and must be formatted as follows: Edges are represented by
+ * tuples of srcVertexId, trgVertexId, distance which are separated by tabs. Edges themselves are
+ * separated by newlines. For example: <code>1\t2\t0.1\n1\t3\t1.4\n</code> defines two edges, edge
+ * 1-2 with distance 0.1, and edge 1-3 with distance 1.4.
*
* If no parameters are provided, the program is run with default data from
* [[SingleSourceShortestPathsData]]
@@ -53,8 +51,10 @@ object SingleSourceShortestPaths {
val graph = Graph.fromDataSet[Long, Double, Double](edges, new InitVertices(srcVertexId), env)
// Execute the scatter-gather iteration
- val result = graph.runScatterGatherIteration(new MinDistanceMessenger,
- new VertexDistanceUpdater, maxIterations)
+ val result = graph.runScatterGatherIteration(
+ new MinDistanceMessenger,
+ new VertexDistanceUpdater,
+ maxIterations)
// Extract the vertices as the result
val singleSourceShortestPaths = result.getVertices
@@ -72,7 +72,7 @@ object SingleSourceShortestPaths {
// Single Source Shortest Path UDFs
// --------------------------------------------------------------------------------------------
- private final class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
+ final private class InitVertices(srcId: Long) extends MapFunction[Long, Double] {
override def map(id: Long) = {
if (id.equals(srcId)) {
@@ -84,11 +84,10 @@ object SingleSourceShortestPaths {
}
/**
- * Distributes the minimum distance associated with a given vertex among all
- * the target vertices summed up with the edge's value.
+ * Distributes the minimum distance associated with a given vertex among all the target vertices
+ * summed up with the edge's value.
*/
- private final class MinDistanceMessenger extends
- ScatterFunction[Long, Double, Double, Double] {
+ final private class MinDistanceMessenger extends ScatterFunction[Long, Double, Double, Double] {
override def sendMessages(vertex: Vertex[Long, Double]) {
if (vertex.getValue < Double.PositiveInfinity) {
@@ -100,10 +99,10 @@ object SingleSourceShortestPaths {
}
/**
- * Function that updates the value of a vertex by picking the minimum
- * distance from all incoming messages.
- */
- private final class VertexDistanceUpdater extends GatherFunction[Long, Double, Double] {
+ * Function that updates the value of a vertex by picking the minimum distance from all incoming
+ * messages.
+ */
+ final private class VertexDistanceUpdater extends GatherFunction[Long, Double, Double] {
override def updateVertex(vertex: Vertex[Long, Double], inMessages: MessageIterator[Double]) {
var minDistance = Double.MaxValue
@@ -130,10 +129,11 @@ object SingleSourceShortestPaths {
private var maxIterations = 5
private def parseParameters(args: Array[String]): Boolean = {
- if(args.length > 0) {
- if(args.length != 4) {
- System.err.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ if (args.length > 0) {
+ if (args.length != 4) {
+ System.err.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
fileOutput = true
srcVertexId = args(0).toLong
@@ -141,26 +141,29 @@ object SingleSourceShortestPaths {
outputPath = args(2)
maxIterations = 3
} else {
- System.out.println("Executing Single Source Shortest Paths example "
- + "with default parameters and built-in default data.")
+ System.out.println(
+ "Executing Single Source Shortest Paths example "
+ + "with default parameters and built-in default data.")
System.out.println(" Provide parameters to read input data from files.")
System.out.println(" See the documentation for the correct format of input files.")
- System.out.println("Usage: SingleSourceShortestPaths <source vertex id>" +
- " <input edges path> <output path> <num iterations>")
+ System.out.println(
+ "Usage: SingleSourceShortestPaths <source vertex id>" +
+ " <input edges path> <output path> <num iterations>")
}
true
}
private def getEdgesDataSet(env: ExecutionEnvironment): DataSet[Edge[Long, Double]] = {
if (fileOutput) {
- env.readCsvFile[(Long, Long, Double)](edgesInputPath,
- lineDelimiter = "\n",
- fieldDelimiter = "\t")
+ env
+ .readCsvFile[(Long, Long, Double)](
+ edgesInputPath,
+ lineDelimiter = "\n",
+ fieldDelimiter = "\t")
.map(new Tuple3ToEdgeMap[Long, Double]())
} else {
- val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES map {
- case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long],
- z.asInstanceOf[Double])
+ val edgeData = SingleSourceShortestPathsData.DEFAULT_EDGES.map {
+ case Array(x, y, z) => (x.asInstanceOf[Long], y.asInstanceOf[Long], z.asInstanceOf[Double])
}
env.fromCollection(edgeData).map(new Tuple3ToEdgeMap[Long, Double]())
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala
index 70a5fdf181a..807e616855f 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunction.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
import org.apache.flink.api.java.tuple.Tuple2
@@ -26,9 +25,11 @@ abstract class EdgesFunction[K, EV, T] extends org.apache.flink.graph.EdgesFunct
def iterateEdges(edges: Iterable[(K, Edge[K, EV])], out: Collector[T])
- override def iterateEdges(edges: java.lang.Iterable[Tuple2[K, Edge[K, EV]]], out:
- Collector[T]): Unit = {
- val scalaIterable = scala.collection.JavaConversions.iterableAsScalaIterable(edges)
+ override def iterateEdges(
+ edges: java.lang.Iterable[Tuple2[K, Edge[K, EV]]],
+ out: Collector[T]): Unit = {
+ val scalaIterable = scala.collection.JavaConversions
+ .iterableAsScalaIterable(edges)
.map(jtuple => (jtuple.f0, jtuple.f1))
iterateEdges(scalaIterable, out)
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala
index 82589b6fbf9..0a67e600cea 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/EdgesFunctionWithVertexValue.scala
@@ -15,19 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.util.Collector
-abstract class EdgesFunctionWithVertexValue[K, VV, EV, T] extends org.apache.flink.graph
-.EdgesFunctionWithVertexValue[K, VV, EV, T] {
+abstract class EdgesFunctionWithVertexValue[K, VV, EV, T]
+ extends org.apache.flink.graph.EdgesFunctionWithVertexValue[K, VV, EV, T] {
@throws(classOf[Exception])
def iterateEdges(v: Vertex[K, VV], edges: Iterable[Edge[K, EV]], out: Collector[T])
- override def iterateEdges(v: Vertex[K, VV], edges: java.lang.Iterable[Edge[K, EV]], out:
- Collector[T]) = {
+ override def iterateEdges(
+ v: Vertex[K, VV],
+ edges: java.lang.Iterable[Edge[K, EV]],
+ out: Collector[T]) = {
iterateEdges(v, scala.collection.JavaConversions.iterableAsScalaIterable(edges), out)
}
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
index df7b8c10822..289d7c57e0b 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/Graph.scala
@@ -15,163 +15,181 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
+import org.apache.flink.{graph => jg}
import org.apache.flink.api.common.functions.{FilterFunction, MapFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.{tuple => jtuple}
import org.apache.flink.api.scala._
import org.apache.flink.graph._
import org.apache.flink.graph.asm.translate.TranslateFunction
-import org.apache.flink.graph.gsa.{ApplyFunction, GSAConfiguration, SumFunction, GatherFunction => GSAGatherFunction}
+import org.apache.flink.graph.gsa.{ApplyFunction, GatherFunction => GSAGatherFunction, GSAConfiguration, SumFunction}
import org.apache.flink.graph.pregel.{ComputeFunction, MessageCombiner, VertexCentricConfiguration}
-import org.apache.flink.graph.spargel.{ScatterFunction, ScatterGatherConfiguration, GatherFunction => SpargelGatherFunction}
+import org.apache.flink.graph.spargel.{GatherFunction => SpargelGatherFunction, ScatterFunction, ScatterGatherConfiguration}
import org.apache.flink.graph.validation.GraphValidator
import org.apache.flink.types.{LongValue, NullValue}
import org.apache.flink.util.Preconditions
-import org.apache.flink.{graph => jg}
import _root_.scala.collection.JavaConverters._
import _root_.scala.reflect.ClassTag
object Graph {
- /**
- * Creates a Graph from a DataSet of vertices and a DataSet of edges.
- */
- def fromDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](vertices: DataSet[Vertex[K, VV]], edges: DataSet[Edge[K, EV]],
- env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ /** Creates a Graph from a DataSet of vertices and a DataSet of edges. */
+ def fromDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ vertices: DataSet[Vertex[K, VV]],
+ edges: DataSet[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
wrapGraph(jg.Graph.fromDataSet[K, VV, EV](vertices.javaSet, edges.javaSet, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of edges.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a DataSet of edges. Vertices are created automatically and their values
+ * are set to NullValue.
*/
- def fromDataSet[K: TypeInformation : ClassTag, EV: TypeInformation : ClassTag]
- (edges: DataSet[Edge[K, EV]], env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
+ def fromDataSet[K: TypeInformation: ClassTag, EV: TypeInformation: ClassTag](
+ edges: DataSet[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
wrapGraph(jg.Graph.fromDataSet[K, EV](edges.javaSet, env.getJavaEnv))
}
/**
- * Creates a graph from a DataSet of edges.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex ids.
+ * Creates a graph from a DataSet of edges. Vertices are created automatically and their values
+ * are set by applying the provided vertexValueInitializer map function to the vertex ids.
*/
- def fromDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](edges: DataSet[Edge[K, EV]],
- vertexValueInitializer: MapFunction[K, VV], env: ExecutionEnvironment): Graph[K, VV, EV] = {
- wrapGraph(jg.Graph.fromDataSet[K, VV, EV](edges.javaSet, vertexValueInitializer,
- env.getJavaEnv))
+ def fromDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ edges: DataSet[Edge[K, EV]],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ wrapGraph(
+ jg.Graph.fromDataSet[K, VV, EV](edges.javaSet, vertexValueInitializer, env.getJavaEnv))
}
- /**
- * Creates a Graph from a Seq of vertices and a Seq of edges.
- */
- def fromCollection[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](vertices: Seq[Vertex[K, VV]], edges: Seq[Edge[K, EV]], env:
- ExecutionEnvironment): Graph[K, VV, EV] = {
- wrapGraph(jg.Graph.fromCollection[K, VV, EV](vertices.asJavaCollection, edges
- .asJavaCollection, env.getJavaEnv))
+ /** Creates a Graph from a Seq of vertices and a Seq of edges. */
+ def fromCollection[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ vertices: Seq[Vertex[K, VV]],
+ edges: Seq[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ wrapGraph(jg.Graph
+ .fromCollection[K, VV, EV](vertices.asJavaCollection, edges.asJavaCollection, env.getJavaEnv))
}
/**
- * Creates a Graph from a Seq of edges.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a Seq of edges. Vertices are created automatically and their values are
+ * set to NullValue.
*/
- def fromCollection[K: TypeInformation : ClassTag, EV: TypeInformation : ClassTag]
- (edges: Seq[Edge[K, EV]], env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
+ def fromCollection[K: TypeInformation: ClassTag, EV: TypeInformation: ClassTag](
+ edges: Seq[Edge[K, EV]],
+ env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
wrapGraph(jg.Graph.fromCollection[K, EV](edges.asJavaCollection, env.getJavaEnv))
}
/**
- * Creates a graph from a Seq of edges.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex ids.
+ * Creates a graph from a Seq of edges. Vertices are created automatically and their values are
+ * set by applying the provided vertexValueInitializer map function to the vertex ids.
*/
- def fromCollection[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](edges: Seq[Edge[K, EV]], vertexValueInitializer: MapFunction[K, VV],
- env: ExecutionEnvironment): Graph[K, VV, EV] = {
- wrapGraph(jg.Graph.fromCollection[K, VV, EV](edges.asJavaCollection, vertexValueInitializer,
- env.getJavaEnv))
+ def fromCollection[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ edges: Seq[Edge[K, EV]],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ wrapGraph(
+ jg.Graph
+ .fromCollection[K, VV, EV](edges.asJavaCollection, vertexValueInitializer, env.getJavaEnv))
}
/**
- * Creates a graph from DataSets of tuples for vertices and for edges.
- * The first field of the Tuple2 vertex object will become the vertex ID
- * and the second field will become the vertex value.
- * The first field of the Tuple3 object for edges will become the source ID,
- * the second field will become the target ID, and the third field will become
- * the edge value.
+ * Creates a graph from DataSets of tuples for vertices and for edges. The first field of the
+ * Tuple2 vertex object will become the vertex ID and the second field will become the vertex
+ * value. The first field of the Tuple3 object for edges will become the source ID, the second
+ * field will become the target ID, and the third field will become the edge value.
*/
- def fromTupleDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](vertices: DataSet[(K, VV)], edges: DataSet[(K, K, EV)],
- env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ def fromTupleDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ vertices: DataSet[(K, VV)],
+ edges: DataSet[(K, K, EV)],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
val javaTupleVertices = vertices.map((v: (K, VV)) => new jtuple.Tuple2(v._1, v._2)).javaSet
val javaTupleEdges = edges.map((v: (K, K, EV)) => new jtuple.Tuple3(v._1, v._2, v._3)).javaSet
- wrapGraph(jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleVertices, javaTupleEdges,
- env.getJavaEnv))
+ wrapGraph(
+ jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleVertices, javaTupleEdges, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuples representing the edges.
- * The first field of the Tuple3 object for edges will become the source ID,
- * the second field will become the target ID, and the third field will become
- * the edge value.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a DataSet of Tuples representing the edges. The first field of the Tuple3
+ * object for edges will become the source ID, the second field will become the target ID, and the
+ * third field will become the edge value. Vertices are created automatically and their values are
+ * set to NullValue.
*/
- def fromTupleDataSet[K: TypeInformation : ClassTag, EV: TypeInformation : ClassTag]
- (edges: DataSet[(K, K, EV)], env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
+ def fromTupleDataSet[K: TypeInformation: ClassTag, EV: TypeInformation: ClassTag](
+ edges: DataSet[(K, K, EV)],
+ env: ExecutionEnvironment): Graph[K, NullValue, EV] = {
val javaTupleEdges = edges.map((v: (K, K, EV)) => new jtuple.Tuple3(v._1, v._2, v._3)).javaSet
wrapGraph(jg.Graph.fromTupleDataSet[K, EV](javaTupleEdges, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuples representing the edges.
- * The first field of the Tuple3 object for edges will become the source ID,
- * the second field will become the target ID, and the third field will become
- * the edge value.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex ids.
+ * Creates a Graph from a DataSet of Tuples representing the edges. The first field of the Tuple3
+ * object for edges will become the source ID, the second field will become the target ID, and the
+ * third field will become the edge value. Vertices are created automatically and their values are
+ * set by applying the provided vertexValueInitializer map function to the vertex ids.
*/
- def fromTupleDataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
- TypeInformation : ClassTag](edges: DataSet[(K, K, EV)],
- vertexValueInitializer: MapFunction[K, VV], env: ExecutionEnvironment): Graph[K, VV, EV] = {
+ def fromTupleDataSet[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
+ edges: DataSet[(K, K, EV)],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, EV] = {
val javaTupleEdges = edges.map((v: (K, K, EV)) => new jtuple.Tuple3(v._1, v._2, v._3)).javaSet
- wrapGraph(jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleEdges, vertexValueInitializer,
- env.getJavaEnv))
+ wrapGraph(
+ jg.Graph.fromTupleDataSet[K, VV, EV](javaTupleEdges, vertexValueInitializer, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuple2's representing the edges.
- * The first field of the Tuple2 object for edges will become the source ID,
- * the second field will become the target ID. The edge value will be set to NullValue.
- * Vertices are created automatically and their values are set to NullValue.
+ * Creates a Graph from a DataSet of Tuple2's representing the edges. The first field of the
+ * Tuple2 object for edges will become the source ID, the second field will become the target ID.
+ * The edge value will be set to NullValue. Vertices are created automatically and their values
+ * are set to NullValue.
*/
- def fromTuple2DataSet[K: TypeInformation : ClassTag](edges: DataSet[(K, K)],
- env: ExecutionEnvironment): Graph[K, NullValue, NullValue] = {
+ def fromTuple2DataSet[K: TypeInformation: ClassTag](
+ edges: DataSet[(K, K)],
+ env: ExecutionEnvironment): Graph[K, NullValue, NullValue] = {
val javaTupleEdges = edges.map((v: (K, K)) => new jtuple.Tuple2(v._1, v._2)).javaSet
wrapGraph(jg.Graph.fromTuple2DataSet[K](javaTupleEdges, env.getJavaEnv))
}
/**
- * Creates a Graph from a DataSet of Tuple2's representing the edges.
- * The first field of the Tuple2 object for edges will become the source ID,
- * the second field will become the target ID. The edge value will be set to NullValue.
- * Vertices are created automatically and their values are set by applying the provided
- * vertexValueInitializer map function to the vertex IDs.
+ * Creates a Graph from a DataSet of Tuple2's representing the edges. The first field of the
+ * Tuple2 object for edges will become the source ID, the second field will become the target ID.
+ * The edge value will be set to NullValue. Vertices are created automatically and their values
+ * are set by applying the provided vertexValueInitializer map function to the vertex IDs.
*/
- def fromTuple2DataSet[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag]
- (edges: DataSet[(K, K)], vertexValueInitializer: MapFunction[K, VV],
- env: ExecutionEnvironment): Graph[K, VV, NullValue] = {
+ def fromTuple2DataSet[K: TypeInformation: ClassTag, VV: TypeInformation: ClassTag](
+ edges: DataSet[(K, K)],
+ vertexValueInitializer: MapFunction[K, VV],
+ env: ExecutionEnvironment): Graph[K, VV, NullValue] = {
val javaTupleEdges = edges.map((v: (K, K)) => new jtuple.Tuple2(v._1, v._2)).javaSet
- wrapGraph(jg.Graph.fromTuple2DataSet[K, VV](javaTupleEdges, vertexValueInitializer,
- env.getJavaEnv))
+ wrapGraph(
+ jg.Graph.fromTuple2DataSet[K, VV](javaTupleEdges, vertexValueInitializer, env.getJavaEnv))
}
- /** Creates a Graph from a CSV file of edges.
+ /**
+ * Creates a Graph from a CSV file of edges.
*
* The edge value is read from the CSV file if [[EV]] is not of type [[NullValue]]. Otherwise the
* edge value is set to [[NullValue]].
@@ -179,50 +197,59 @@ object Graph {
* If the vertex value type [[VV]] is specified (unequal [[NullValue]]), then the vertex values
* are read from the file specified by pathVertices. If the path has not been specified then the
* vertexValueInitializer is used to initialize the vertex values of the vertices extracted from
- * the set of edges. If the vertexValueInitializer has not been set either, then the method
- * fails.
+ * the set of edges. If the vertexValueInitializer has not been set either, then the method fails.
*
- * @param env The Execution Environment.
- * @param pathEdges The file path containing the edges.
- * @param pathVertices The file path containing the vertices.
- * @param lineDelimiterVertices The string that separates lines in the vertices file. It defaults
- * to newline.
- * @param fieldDelimiterVertices The string that separates vertex Ids from vertex values in the
- * vertices file.
- * @param quoteCharacterVertices The character to use for quoted String parsing in the vertices
- * file. Disabled by default.
- * @param ignoreFirstLineVertices Whether the first line in the vertices file should be ignored.
- * @param ignoreCommentsVertices Lines that start with the given String in the vertices file
- * are ignored, disabled by default.
- * @param lenientVertices Whether the parser should silently ignore malformed lines in the
- * vertices file.
- * @param includedFieldsVertices The fields in the vertices file that should be read. By default
- * all fields are read.
- * @param lineDelimiterEdges The string that separates lines in the edges file. It defaults to
- * newline.
- * @param fieldDelimiterEdges The string that separates fields in the edges file.
- * @param quoteCharacterEdges The character to use for quoted String parsing in the edges file.
- * Disabled by default.
- * @param ignoreFirstLineEdges Whether the first line in the vertices file should be ignored.
- * @param ignoreCommentsEdges Lines that start with the given String in the edges file are
- * ignored, disabled by default.
- * @param lenientEdges Whether the parser should silently ignore malformed lines in the edges
- * file.
- * @param includedFieldsEdges The fields in the edges file that should be read. By default all
- * fields are read.
- * @param vertexValueInitializer If no vertex values are provided, this mapper can be used to
- * initialize them, by applying a map transformation on the vertex
- * IDs.
- * @tparam K Vertex key type
- * @tparam VV Vertex value type
- * @tparam EV Edge value type
- * @return Graph with vertices and edges read from the given files.
- */
- // This method exceeds the max allowed number of parameters -->
+ * @param env
+ * The Execution Environment.
+ * @param pathEdges
+ * The file path containing the edges.
+ * @param pathVertices
+ * The file path containing the vertices.
+ * @param lineDelimiterVertices
+ * The string that separates lines in the vertices file. It defaults to newline.
+ * @param fieldDelimiterVertices
+ * The string that separates vertex Ids from vertex values in the vertices file.
+ * @param quoteCharacterVertices
+ * The character to use for quoted String parsing in the vertices file. Disabled by default.
+ * @param ignoreFirstLineVertices
+ * Whether the first line in the vertices file should be ignored.
+ * @param ignoreCommentsVertices
+ * Lines that start with the given String in the vertices file are ignored, disabled by default.
+ * @param lenientVertices
+ * Whether the parser should silently ignore malformed lines in the vertices file.
+ * @param includedFieldsVertices
+ * The fields in the vertices file that should be read. By default all fields are read.
+ * @param lineDelimiterEdges
+ * The string that separates lines in the edges file. It defaults to newline.
+ * @param fieldDelimiterEdges
+ * The string that separates fields in the edges file.
+ * @param quoteCharacterEdges
+ * The character to use for quoted String parsing in the edges file. Disabled by default.
+ * @param ignoreFirstLineEdges
+ * Whether the first line in the vertices file should be ignored.
+ * @param ignoreCommentsEdges
+ * Lines that start with the given String in the edges file are ignored, disabled by default.
+ * @param lenientEdges
+ * Whether the parser should silently ignore malformed lines in the edges file.
+ * @param includedFieldsEdges
+ * The fields in the edges file that should be read. By default all fields are read.
+ * @param vertexValueInitializer
+ * If no vertex values are provided, this mapper can be used to initialize them, by applying a
+ * map transformation on the vertex IDs.
+ * @tparam K
+ * Vertex key type
+ * @tparam VV
+ * Vertex value type
+ * @tparam EV
+ * Edge value type
+ * @return
+ * Graph with vertices and edges read from the given files.
+ */
+ // This method exceeds the max allowed number of parameters -->
def fromCsvReader[
- K: TypeInformation : ClassTag,
- VV: TypeInformation : ClassTag,
- EV: TypeInformation : ClassTag](
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](
env: ExecutionEnvironment,
pathEdges: String,
pathVertices: String = null,
@@ -240,8 +267,7 @@ object Graph {
ignoreCommentsEdges: String = null,
lenientEdges: Boolean = false,
includedFieldsEdges: Array[Int] = null,
- vertexValueInitializer: MapFunction[K, VV] = null)
- : Graph[K, VV, EV] = {
+ vertexValueInitializer: MapFunction[K, VV] = null): Graph[K, VV, EV] = {
Preconditions.checkNotNull(pathEdges)
@@ -249,15 +275,16 @@ object Graph {
val vvClassTag = implicitly[ClassTag[VV]]
val edges = if (evClassTag.runtimeClass.equals(classOf[NullValue])) {
- env.readCsvFile[(K, K)](
- pathEdges,
- lineDelimiterEdges,
- fieldDelimiterEdges,
- quoteCharacterEdges,
- ignoreFirstLineEdges,
- ignoreCommentsEdges,
- lenientEdges,
- includedFieldsEdges)
+ env
+ .readCsvFile[(K, K)](
+ pathEdges,
+ lineDelimiterEdges,
+ fieldDelimiterEdges,
+ quoteCharacterEdges,
+ ignoreFirstLineEdges,
+ ignoreCommentsEdges,
+ lenientEdges,
+ includedFieldsEdges)
.map((edge: (K, K)) => (edge._1, edge._2, NullValue.getInstance))
.asInstanceOf[DataSet[(K, K, EV)]]
} else {
@@ -276,16 +303,24 @@ object Graph {
fromTupleDataSet[K, EV](edges, env).asInstanceOf[Graph[K, VV, EV]]
} else {
if (pathVertices != null) {
- val vertices = env.readCsvFile[(K, VV)](pathVertices, lineDelimiterVertices,
- fieldDelimiterVertices, quoteCharacterVertices, ignoreFirstLineVertices,
- ignoreCommentsVertices, lenientVertices, includedFieldsVertices)
+ val vertices = env.readCsvFile[(K, VV)](
+ pathVertices,
+ lineDelimiterVertices,
+ fieldDelimiterVertices,
+ quoteCharacterVertices,
+ ignoreFirstLineVertices,
+ ignoreCommentsVertices,
+ lenientVertices,
+ includedFieldsVertices
+ )
fromTupleDataSet[K, VV, EV](vertices, edges, env)
} else if (vertexValueInitializer != null) {
fromTupleDataSet[K, VV, EV](edges, vertexValueInitializer, env)
} else {
- throw new IllegalArgumentException("Path vertices path and vertex value initialzier must" +
- "not be null if the vertex value type is not NullValue.")
+ throw new IllegalArgumentException(
+ "Path vertices path and vertex value initialzier must" +
+ "not be null if the vertex value type is not NullValue.")
}
}
}
@@ -294,19 +329,26 @@ object Graph {
/**
* Represents a graph consisting of [[Edge]] edges and [[Vertex]] vertices.
*
- * @param jgraph the underlying java api Graph.
- * @tparam K the key type for vertex and edge identifiers
- * @tparam VV the value type for vertices
- * @tparam EV the value type for edges
- * @see org.apache.flink.graph.Edge
- * @see org.apache.flink.graph.Vertex
+ * @param jgraph
+ * the underlying java api Graph.
+ * @tparam K
+ * the key type for vertex and edge identifiers
+ * @tparam VV
+ * the value type for vertices
+ * @tparam EV
+ * the value type for edges
+ * @see
+ * org.apache.flink.graph.Edge
+ * @see
+ * org.apache.flink.graph.Vertex
*/
-final class Graph[K: TypeInformation : ClassTag, VV: TypeInformation : ClassTag, EV:
-TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
+final class Graph[
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](jgraph: jg.Graph[K, VV, EV]) {
private[flink] def getWrappedGraph = jgraph
-
private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
if (jgraph.getContext.getConfig.isClosureCleanerEnabled) {
ClosureCleaner.clean(f, checkSerializable, jgraph.getContext.getConfig.getClosureCleanerLevel)
@@ -315,36 +357,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
f
}
- /**
- * @return the vertex DataSet.
- */
+ /** @return the vertex DataSet. */
def getVertices = wrap(jgraph.getVertices)
- /**
- * @return the edge DataSet.
- */
+ /** @return the edge DataSet. */
def getEdges = wrap(jgraph.getEdges)
- /**
- * @return the vertex DataSet as Tuple2.
- */
+ /** @return the vertex DataSet as Tuple2. */
def getVerticesAsTuple2(): DataSet[(K, VV)] = {
- wrap(jgraph.getVerticesAsTuple2).map(
- (v: jtuple.Tuple2[K, VV]) => (v.f0, v.f1))
+ wrap(jgraph.getVerticesAsTuple2).map((v: jtuple.Tuple2[K, VV]) => (v.f0, v.f1))
}
- /**
- * @return the edge DataSet as Tuple3.
- */
+ /** @return the edge DataSet as Tuple3. */
def getEdgesAsTuple3(): DataSet[(K, K, EV)] = {
- wrap(jgraph.getEdgesAsTuple3).map(
- (e: jtuple.Tuple3[K, K, EV]) =>
- (e.f0, e.f1, e.f2))
+ wrap(jgraph.getEdgesAsTuple3).map((e: jtuple.Tuple3[K, K, EV]) => (e.f0, e.f1, e.f2))
}
/**
- * @return a DataSet of Triplets,
- * consisting of (srcVertexId, trgVertexId, srcVertexValue, trgVertexValue, edgeValue)
+ * @return
+ * a DataSet of Triplets, consisting of (srcVertexId, trgVertexId, srcVertexValue,
+ * trgVertexValue, edgeValue)
*/
def getTriplets(): DataSet[Triplet[K, VV, EV]] = {
wrap(jgraph.getTriplets())
@@ -353,24 +385,29 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Apply a function to the attribute of each vertex in the graph.
*
- * @param mapper the map function to apply.
- * @return a new graph
+ * @param mapper
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapVertices[NV: TypeInformation : ClassTag](mapper: MapFunction[Vertex[K, VV], NV]):
- Graph[K, NV, EV] = {
- new Graph[K, NV, EV](jgraph.mapVertices[NV](
- mapper,
- createTypeInformation[Vertex[K, NV]]
- ))
+ def mapVertices[NV: TypeInformation: ClassTag](
+ mapper: MapFunction[Vertex[K, VV], NV]): Graph[K, NV, EV] = {
+ new Graph[K, NV, EV](
+ jgraph.mapVertices[NV](
+ mapper,
+ createTypeInformation[Vertex[K, NV]]
+ ))
}
/**
* Apply a function to the attribute of each vertex in the graph.
*
- * @param fun the map function to apply.
- * @return a new graph
+ * @param fun
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapVertices[NV: TypeInformation : ClassTag](fun: Vertex[K, VV] => NV): Graph[K, NV, EV] = {
+ def mapVertices[NV: TypeInformation: ClassTag](fun: Vertex[K, VV] => NV): Graph[K, NV, EV] = {
val mapper: MapFunction[Vertex[K, VV], NV] = new MapFunction[Vertex[K, VV], NV] {
val cleanFun = clean(fun)
@@ -382,24 +419,29 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Apply a function to the attribute of each edge in the graph.
*
- * @param mapper the map function to apply.
- * @return a new graph
+ * @param mapper
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapEdges[NV: TypeInformation : ClassTag](mapper: MapFunction[Edge[K, EV], NV]): Graph[K,
- VV, NV] = {
- new Graph[K, VV, NV](jgraph.mapEdges[NV](
- mapper,
- createTypeInformation[Edge[K, NV]]
- ))
+ def mapEdges[NV: TypeInformation: ClassTag](
+ mapper: MapFunction[Edge[K, EV], NV]): Graph[K, VV, NV] = {
+ new Graph[K, VV, NV](
+ jgraph.mapEdges[NV](
+ mapper,
+ createTypeInformation[Edge[K, NV]]
+ ))
}
/**
* Apply a function to the attribute of each edge in the graph.
*
- * @param fun the map function to apply.
- * @return a new graph
+ * @param fun
+ * the map function to apply.
+ * @return
+ * a new graph
*/
- def mapEdges[NV: TypeInformation : ClassTag](fun: Edge[K, EV] => NV): Graph[K, VV, NV] = {
+ def mapEdges[NV: TypeInformation: ClassTag](fun: Edge[K, EV] => NV): Graph[K, VV, NV] = {
val mapper: MapFunction[Edge[K, EV], NV] = new MapFunction[Edge[K, EV], NV] {
val cleanFun = clean(fun)
@@ -411,22 +453,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Translate vertex and edge IDs using the given MapFunction.
*
- * @param translator implements conversion from K to NEW
- * @return graph with translated vertex and edge IDs
+ * @param translator
+ * implements conversion from K to NEW
+ * @return
+ * graph with translated vertex and edge IDs
*/
- def translateGraphIds[NEW: TypeInformation : ClassTag](translator: TranslateFunction[K, NEW]):
- Graph[NEW, VV, EV] = {
+ def translateGraphIds[NEW: TypeInformation: ClassTag](
+ translator: TranslateFunction[K, NEW]): Graph[NEW, VV, EV] = {
new Graph[NEW, VV, EV](jgraph.translateGraphIds(translator))
}
/**
* Translate vertex and edge IDs using the given function.
*
- * @param fun implements conversion from K to NEW
- * @return graph with translated vertex and edge IDs
+ * @param fun
+ * implements conversion from K to NEW
+ * @return
+ * graph with translated vertex and edge IDs
*/
- def translateGraphIds[NEW: TypeInformation : ClassTag](fun: (K, NEW) => NEW):
- Graph[NEW, VV, EV] = {
+ def translateGraphIds[NEW: TypeInformation: ClassTag](
+ fun: (K, NEW) => NEW): Graph[NEW, VV, EV] = {
val translator: TranslateFunction[K, NEW] = new TranslateFunction[K, NEW] {
val cleanFun = clean(fun)
@@ -439,22 +485,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Translate vertex values using the given MapFunction.
*
- * @param translator implements conversion from VV to NEW
- * @return graph with translated vertex values
+ * @param translator
+ * implements conversion from VV to NEW
+ * @return
+ * graph with translated vertex values
*/
- def translateVertexValues[NEW: TypeInformation : ClassTag](translator:
- TranslateFunction[VV, NEW]): Graph[K, NEW, EV] = {
+ def translateVertexValues[NEW: TypeInformation: ClassTag](
+ translator: TranslateFunction[VV, NEW]): Graph[K, NEW, EV] = {
new Graph[K, NEW, EV](jgraph.translateVertexValues(translator))
}
/**
* Translate vertex values using the given function.
*
- * @param fun implements conversion from VV to NEW
- * @return graph with translated vertex values
+ * @param fun
+ * implements conversion from VV to NEW
+ * @return
+ * graph with translated vertex values
*/
- def translateVertexValues[NEW: TypeInformation : ClassTag](fun: (VV, NEW) => NEW):
- Graph[K, NEW, EV] = {
+ def translateVertexValues[NEW: TypeInformation: ClassTag](
+ fun: (VV, NEW) => NEW): Graph[K, NEW, EV] = {
val translator: TranslateFunction[VV, NEW] = new TranslateFunction[VV, NEW] {
val cleanFun = clean(fun)
@@ -467,22 +517,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Translate edge values using the given MapFunction.
*
- * @param translator implements conversion from EV to NEW
- * @return graph with translated edge values
+ * @param translator
+ * implements conversion from EV to NEW
+ * @return
+ * graph with translated edge values
*/
- def translateEdgeValues[NEW: TypeInformation : ClassTag](translator: TranslateFunction[EV, NEW]):
- Graph[K, VV, NEW] = {
+ def translateEdgeValues[NEW: TypeInformation: ClassTag](
+ translator: TranslateFunction[EV, NEW]): Graph[K, VV, NEW] = {
new Graph[K, VV, NEW](jgraph.translateEdgeValues(translator))
}
/**
* Translate edge values using the given function.
*
- * @param fun implements conversion from EV to NEW
- * @return graph with translated edge values
+ * @param fun
+ * implements conversion from EV to NEW
+ * @return
+ * graph with translated edge values
*/
- def translateEdgeValues[NEW: TypeInformation : ClassTag](fun: (EV, NEW) => NEW):
- Graph[K, VV, NEW] = {
+ def translateEdgeValues[NEW: TypeInformation: ClassTag](
+ fun: (EV, NEW) => NEW): Graph[K, VV, NEW] = {
val translator: TranslateFunction[EV, NEW] = new TranslateFunction[EV, NEW] {
val cleanFun = clean(fun)
@@ -493,44 +547,49 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies
- * a user-defined transformation on the values of the matched records.
- * The vertex ID and the first field of the Tuple2 DataSet are used as the join keys.
- *
- * @param inputDataSet the Tuple2 DataSet to join with.
- * The first field of the Tuple2 is used as the join key and the second field is passed
- * as a parameter to the transformation function.
- * @param vertexJoinFunction the transformation function to apply.
- * The first parameter is the current vertex value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @return a new Graph, where the vertex values have been updated according to the
- * result of the vertexJoinFunction.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- */
- def joinWithVertices[T: TypeInformation](inputDataSet: DataSet[(K, T)],
- vertexJoinFunction: VertexJoinFunction[VV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies a user-defined
+ * transformation on the values of the matched records. The vertex ID and the first field of the
+ * Tuple2 DataSet are used as the join keys.
+ *
+ * @param inputDataSet
+ * the Tuple2 DataSet to join with. The first field of the Tuple2 is used as the join key and
+ * the second field is passed as a parameter to the transformation function.
+ * @param vertexJoinFunction
+ * the transformation function to apply. The first parameter is the current vertex value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @return
+ * a new Graph, where the vertex values have been updated according to the result of the
+ * vertexJoinFunction.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ */
+ def joinWithVertices[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ vertexJoinFunction: VertexJoinFunction[VV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithVertices[T](javaTupleSet, vertexJoinFunction))
}
/**
- * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies
- * a user-defined transformation on the values of the matched records.
- * The vertex ID and the first field of the Tuple2 DataSet are used as the join keys.
- *
- * @param inputDataSet the Tuple2 DataSet to join with.
- * The first field of the Tuple2 is used as the join key and the second field is passed
- * as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current vertex value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @return a new Graph, where the vertex values have been updated according to the
- * result of the vertexJoinFunction.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- */
- def joinWithVertices[T: TypeInformation](inputDataSet: DataSet[(K, T)], fun: (VV, T) => VV):
- Graph[K, VV, EV] = {
+ * Joins the vertex DataSet of this graph with an input Tuple2 DataSet and applies a user-defined
+ * transformation on the values of the matched records. The vertex ID and the first field of the
+ * Tuple2 DataSet are used as the join keys.
+ *
+ * @param inputDataSet
+ * the Tuple2 DataSet to join with. The first field of the Tuple2 is used as the join key and
+ * the second field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current vertex value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @return
+ * a new Graph, where the vertex values have been updated according to the result of the
+ * vertexJoinFunction.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ */
+ def joinWithVertices[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ fun: (VV, T) => VV): Graph[K, VV, EV] = {
val newVertexJoin = new VertexJoinFunction[VV, T]() {
val cleanFun = clean(fun)
@@ -538,50 +597,55 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
cleanFun(vertexValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithVertices[T](javaTupleSet, newVertexJoin))
}
/**
- * Joins the edge DataSet with an input DataSet on the composite key of both
- * source and target IDs and applies a user-defined transformation on the values
- * of the matched records. The first two fields of the input DataSet are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first two fields of the Tuple3 are used as the composite join key
- * and the third field is passed as a parameter to the transformation function.
- * @param edgeJoinFunction the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple3 from the input DataSet.
- * @tparam T the type of the third field of the input Tuple3 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdges[T: TypeInformation](inputDataSet: DataSet[(K, K, T)],
- edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
+ * Joins the edge DataSet with an input DataSet on the composite key of both source and target IDs
+ * and applies a user-defined transformation on the values of the matched records. The first two
+ * fields of the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first two fields of the Tuple3 are used as the composite join
+ * key and the third field is passed as a parameter to the transformation function.
+ * @param edgeJoinFunction
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple3 from the input DataSet.
+ * @tparam T
+ * the type of the third field of the input Tuple3 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdges[T: TypeInformation](
+ inputDataSet: DataSet[(K, K, T)],
+ edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet =
+ inputDataSet.map((i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
wrapGraph(jgraph.joinWithEdges[T](javaTupleSet, edgeJoinFunction))
}
/**
- * Joins the edge DataSet with an input DataSet on the composite key of both
- * source and target IDs and applies a user-defined transformation on the values
- * of the matched records. The first two fields of the input DataSet are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first two fields of the Tuple3 are used as the composite join key
- * and the third field is passed as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple3 from the input DataSet.
- * @tparam T the type of the third field of the input Tuple3 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdges[T: TypeInformation](inputDataSet: DataSet[(K, K, T)], fun: (EV, T) => EV):
- Graph[K, VV, EV] = {
+ * Joins the edge DataSet with an input DataSet on the composite key of both source and target IDs
+ * and applies a user-defined transformation on the values of the matched records. The first two
+ * fields of the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first two fields of the Tuple3 are used as the composite join
+ * key and the third field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple3 from the input DataSet.
+ * @tparam T
+ * the type of the third field of the input Tuple3 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdges[T: TypeInformation](
+ inputDataSet: DataSet[(K, K, T)],
+ fun: (EV, T) => EV): Graph[K, VV, EV] = {
val newEdgeJoin = new EdgeJoinFunction[EV, T]() {
val cleanFun = clean(fun)
@@ -589,52 +653,55 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
cleanFun(edgeValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
+ val javaTupleSet =
+ inputDataSet.map((i: (K, K, T)) => new jtuple.Tuple3(i._1, i._2, i._3)).javaSet
wrapGraph(jgraph.joinWithEdges[T](javaTupleSet, newEdgeJoin))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The source ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param edgeJoinFunction the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnSource[T: TypeInformation](inputDataSet: DataSet[(K, T)],
- edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ * on the values of the matched records. The source ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param edgeJoinFunction
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnSource[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnSource[T](javaTupleSet, edgeJoinFunction))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The source ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnSource[T: TypeInformation](inputDataSet: DataSet[(K, T)], fun: (EV, T) =>
- EV): Graph[K, VV, EV] = {
+ * on the values of the matched records. The source ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnSource[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ fun: (EV, T) => EV): Graph[K, VV, EV] = {
val newEdgeJoin = new EdgeJoinFunction[EV, T]() {
val cleanFun = clean(fun)
@@ -642,87 +709,94 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
cleanFun(edgeValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnSource[T](javaTupleSet, newEdgeJoin))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The target ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param edgeJoinFunction the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnTarget[T: TypeInformation](inputDataSet: DataSet[(K, T)],
- edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ * on the values of the matched records. The target ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param edgeJoinFunction
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnTarget[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ edgeJoinFunction: EdgeJoinFunction[EV, T]): Graph[K, VV, EV] = {
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnTarget[T](javaTupleSet, edgeJoinFunction))
}
/**
* Joins the edge DataSet with an input Tuple2 DataSet and applies a user-defined transformation
- * on the values of the matched records.
- * The target ID of the edges input and the first field of the input DataSet
- * are used as join keys.
- *
- * @param inputDataSet the DataSet to join with.
- * The first field of the Tuple2 is used as the join key
- * and the second field is passed as a parameter to the transformation function.
- * @param fun the transformation function to apply.
- * The first parameter is the current edge value and the second parameter is the value
- * of the matched Tuple2 from the input DataSet.
- * @tparam T the type of the second field of the input Tuple2 DataSet.
- * @return a new Graph, where the edge values have been updated according to the
- * result of the edgeJoinFunction.
- */
- def joinWithEdgesOnTarget[T: TypeInformation](inputDataSet: DataSet[(K, T)], fun: (EV, T) =>
- EV): Graph[K, VV, EV] = {
+ * on the values of the matched records. The target ID of the edges input and the first field of
+ * the input DataSet are used as join keys.
+ *
+ * @param inputDataSet
+ * the DataSet to join with. The first field of the Tuple2 is used as the join key and the
+ * second field is passed as a parameter to the transformation function.
+ * @param fun
+ * the transformation function to apply. The first parameter is the current edge value and the
+ * second parameter is the value of the matched Tuple2 from the input DataSet.
+ * @tparam T
+ * the type of the second field of the input Tuple2 DataSet.
+ * @return
+ * a new Graph, where the edge values have been updated according to the result of the
+ * edgeJoinFunction.
+ */
+ def joinWithEdgesOnTarget[T: TypeInformation](
+ inputDataSet: DataSet[(K, T)],
+ fun: (EV, T) => EV): Graph[K, VV, EV] = {
val newEdgeJoin = new EdgeJoinFunction[EV, T]() {
val cleanFun = clean(fun)
- override def edgeJoin(edgeValue: EV, inputValue:T): EV = {
+ override def edgeJoin(edgeValue: EV, inputValue: T): EV = {
cleanFun(edgeValue, inputValue)
}
}
- val javaTupleSet = inputDataSet.map(
- (i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
+ val javaTupleSet = inputDataSet.map((i: (K, T)) => new jtuple.Tuple2(i._1, i._2)).javaSet
wrapGraph(jgraph.joinWithEdgesOnTarget[T](javaTupleSet, newEdgeJoin))
}
/**
- * Apply filtering functions to the graph and return a sub-graph that
- * satisfies the predicates for both vertices and edges.
+ * Apply filtering functions to the graph and return a sub-graph that satisfies the predicates for
+ * both vertices and edges.
*
- * @param vertexFilter the filter function for vertices.
- * @param edgeFilter the filter function for edges.
- * @return the resulting sub-graph.
- */
- def subgraph(vertexFilter: FilterFunction[Vertex[K, VV]], edgeFilter: FilterFunction[Edge[K,
- EV]]) = {
+ * @param vertexFilter
+ * the filter function for vertices.
+ * @param edgeFilter
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
+ */
+ def subgraph(
+ vertexFilter: FilterFunction[Vertex[K, VV]],
+ edgeFilter: FilterFunction[Edge[K, EV]]) = {
wrapGraph(jgraph.subgraph(vertexFilter, edgeFilter))
}
/**
- * Apply filtering functions to the graph and return a sub-graph that
- * satisfies the predicates for both vertices and edges.
+ * Apply filtering functions to the graph and return a sub-graph that satisfies the predicates for
+ * both vertices and edges.
*
- * @param vertexFilterFun the filter function for vertices.
- * @param edgeFilterFun the filter function for edges.
- * @return the resulting sub-graph.
- */
- def subgraph(vertexFilterFun: Vertex[K, VV] => Boolean, edgeFilterFun: Edge[K, EV] =>
- Boolean) = {
+ * @param vertexFilterFun
+ * the filter function for vertices.
+ * @param edgeFilterFun
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
+ */
+ def subgraph(vertexFilterFun: Vertex[K, VV] => Boolean, edgeFilterFun: Edge[K, EV] => Boolean) = {
val vertexFilter = new FilterFunction[Vertex[K, VV]] {
val cleanVertexFun = clean(vertexFilterFun)
@@ -739,22 +813,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the vertices.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the vertices.
*
- * @param vertexFilter the filter function for vertices.
- * @return the resulting sub-graph.
+ * @param vertexFilter
+ * the filter function for vertices.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnVertices(vertexFilter: FilterFunction[Vertex[K, VV]]) = {
wrapGraph(jgraph.filterOnVertices(vertexFilter))
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the vertices.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the vertices.
*
- * @param vertexFilterFun the filter function for vertices.
- * @return the resulting sub-graph.
+ * @param vertexFilterFun
+ * the filter function for vertices.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnVertices(vertexFilterFun: Vertex[K, VV] => Boolean) = {
val vertexFilter = new FilterFunction[Vertex[K, VV]] {
@@ -767,22 +845,26 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the edges.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the edges.
*
- * @param edgeFilter the filter function for edges.
- * @return the resulting sub-graph.
+ * @param edgeFilter
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnEdges(edgeFilter: FilterFunction[Edge[K, EV]]) = {
wrapGraph(jgraph.filterOnEdges(edgeFilter))
}
/**
- * Apply a filtering function to the graph and return a sub-graph that
- * satisfies the predicates only for the edges.
+ * Apply a filtering function to the graph and return a sub-graph that satisfies the predicates
+ * only for the edges.
*
- * @param edgeFilterFun the filter function for edges.
- * @return the resulting sub-graph.
+ * @param edgeFilterFun
+ * the filter function for edges.
+ * @return
+ * the resulting sub-graph.
*/
def filterOnEdges(edgeFilterFun: Edge[K, EV] => Boolean) = {
val edgeFilter = new FilterFunction[Edge[K, EV]] {
@@ -797,7 +879,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Return the in-degree of all vertices in the graph
*
- * @return A DataSet of Tuple2<vertexId, inDegree>
+ * @return
+ * A DataSet of Tuple2<vertexId, inDegree>
*/
def inDegrees(): DataSet[(K, LongValue)] = {
wrap(jgraph.inDegrees).map((i: jtuple.Tuple2[K, LongValue]) => (i.f0, i.f1))
@@ -806,7 +889,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Return the out-degree of all vertices in the graph
*
- * @return A DataSet of Tuple2<vertexId, outDegree>
+ * @return
+ * A DataSet of Tuple2<vertexId, outDegree>
*/
def outDegrees(): DataSet[(K, LongValue)] = {
wrap(jgraph.outDegrees).map((i: jtuple.Tuple2[K, LongValue]) => (i.f0, i.f1))
@@ -815,7 +899,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Return the degree of all vertices in the graph
*
- * @return A DataSet of Tuple2<vertexId, degree>
+ * @return
+ * A DataSet of Tuple2<vertexId, degree>
*/
def getDegrees(): DataSet[(K, LongValue)] = {
wrap(jgraph.getDegrees).map((i: jtuple.Tuple2[K, LongValue]) => (i.f0, i.f1))
@@ -824,7 +909,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* This operation adds all inverse-direction edges to the graph.
*
- * @return the undirected graph.
+ * @return
+ * the undirected graph.
*/
def getUndirected(): Graph[K, VV, EV] = {
new Graph(jgraph.getUndirected)
@@ -833,7 +919,8 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Reverse the direction of the edges in the graph
*
- * @return a new graph with all edges reversed
+ * @return
+ * a new graph with all edges reversed
* @throws UnsupportedOperationException
*/
def reverse(): Graph[K, VV, EV] = {
@@ -841,113 +928,121 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
}
/**
- * Compute an aggregate over the edges of each vertex. The function applied
- * on the edges has access to the vertex value.
+ * Compute an aggregate over the edges of each vertex. The function applied on the edges has
+ * access to the vertex value.
*
- * @param edgesFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnEdges[T: TypeInformation : ClassTag](edgesFunction:
- EdgesFunctionWithVertexValue[K, VV, EV,
- T], direction: EdgeDirection):
- DataSet[T] = {
+ * @param edgesFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnEdges[T: TypeInformation: ClassTag](
+ edgesFunction: EdgesFunctionWithVertexValue[K, VV, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnEdges(edgesFunction, direction, createTypeInformation[T]))
}
/**
- * Compute an aggregate over the edges of each vertex. The function applied
- * on the edges has access to the vertex value.
+ * Compute an aggregate over the edges of each vertex. The function applied on the edges has
+ * access to the vertex value.
*
- * @param edgesFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnEdges[T: TypeInformation : ClassTag](edgesFunction: EdgesFunction[K, EV, T],
- direction: EdgeDirection): DataSet[T] = {
+ * @param edgesFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnEdges[T: TypeInformation: ClassTag](
+ edgesFunction: EdgesFunction[K, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnEdges(edgesFunction, direction, createTypeInformation[T]))
}
/**
- * Compute an aggregate over the neighbors (edges and vertices) of each
- * vertex. The function applied on the neighbors has access to the vertex
- * value.
+ * Compute an aggregate over the neighbors (edges and vertices) of each vertex. The function
+ * applied on the neighbors has access to the vertex value.
*
- * @param neighborsFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnNeighbors[T: TypeInformation : ClassTag](neighborsFunction:
- NeighborsFunctionWithVertexValue[K,
- VV, EV, T], direction:
- EdgeDirection): DataSet[T] = {
+ * @param neighborsFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnNeighbors[T: TypeInformation: ClassTag](
+ neighborsFunction: NeighborsFunctionWithVertexValue[K, VV, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnNeighbors(neighborsFunction, direction, createTypeInformation[T]))
}
/**
- * Compute an aggregate over the neighbors (edges and vertices) of each
- * vertex.
+ * Compute an aggregate over the neighbors (edges and vertices) of each vertex.
*
- * @param neighborsFunction the function to apply to the neighborhood
- * @param direction the edge direction (in-, out-, all-)
- * @tparam T the output type
- * @return a dataset of a T
- */
- def groupReduceOnNeighbors[T: TypeInformation : ClassTag](neighborsFunction:
- NeighborsFunction[K, VV, EV, T],
- direction: EdgeDirection):
- DataSet[T] = {
+ * @param neighborsFunction
+ * the function to apply to the neighborhood
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @tparam T
+ * the output type
+ * @return
+ * a dataset of a T
+ */
+ def groupReduceOnNeighbors[T: TypeInformation: ClassTag](
+ neighborsFunction: NeighborsFunction[K, VV, EV, T],
+ direction: EdgeDirection): DataSet[T] = {
wrap(jgraph.groupReduceOnNeighbors(neighborsFunction, direction, createTypeInformation[T]))
}
- /**
- * @return a long integer representing the number of vertices
- */
+ /** @return a long integer representing the number of vertices */
def numberOfVertices(): Long = {
jgraph.numberOfVertices()
}
- /**
- * @return a long integer representing the number of edges
- */
+ /** @return a long integer representing the number of edges */
def numberOfEdges(): Long = {
jgraph.numberOfEdges()
}
- /**
- * @return The IDs of the vertices as DataSet
- */
+ /** @return The IDs of the vertices as DataSet */
def getVertexIds(): DataSet[K] = {
wrap(jgraph.getVertexIds)
}
- /**
- * @return The IDs of the edges as DataSet
- */
+ /** @return The IDs of the edges as DataSet */
def getEdgeIds(): DataSet[(K, K)] = {
wrap(jgraph.getEdgeIds).map((i: jtuple.Tuple2[K, K]) => (i.f0, i.f1))
}
/**
- * Adds the input vertex to the graph. If the vertex already
- * exists in the graph, it will not be added again.
+ * Adds the input vertex to the graph. If the vertex already exists in the graph, it will not be
+ * added again.
*
- * @param vertex the vertex to be added
- * @return the new graph containing the existing vertices as well as the one just added
+ * @param vertex
+ * the vertex to be added
+ * @return
+ * the new graph containing the existing vertices as well as the one just added
*/
def addVertex(vertex: Vertex[K, VV]) = {
wrapGraph(jgraph.addVertex(vertex))
}
/**
- * Adds the list of vertices, passed as input, to the graph.
- * If the vertices already exist in the graph, they will not be added once more.
+ * Adds the list of vertices, passed as input, to the graph. If the vertices already exist in the
+ * graph, they will not be added once more.
*
- * @param vertices the list of vertices to add
- * @return the new graph containing the existing and newly added vertices
+ * @param vertices
+ * the list of vertices to add
+ * @return
+ * the new graph containing the existing and newly added vertices
*/
def addVertices(vertices: List[Vertex[K, VV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.addVertices(vertices.asJava))
@@ -956,25 +1051,30 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Adds the given list edges to the graph.
*
- * When adding an edge for a non-existing set of vertices,
- * the edge is considered invalid and ignored.
+ * When adding an edge for a non-existing set of vertices, the edge is considered invalid and
+ * ignored.
*
- * @param edges the data set of edges to be added
- * @return a new graph containing the existing edges plus the newly added edges.
+ * @param edges
+ * the data set of edges to be added
+ * @return
+ * a new graph containing the existing edges plus the newly added edges.
*/
def addEdges(edges: List[Edge[K, EV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.addEdges(edges.asJava))
}
/**
- * Adds the given edge to the graph. If the source and target vertices do
- * not exist in the graph, they will also be added.
+ * Adds the given edge to the graph. If the source and target vertices do not exist in the graph,
+ * they will also be added.
*
- * @param source the source vertex of the edge
- * @param target the target vertex of the edge
- * @param edgeValue the edge value
- * @return the new graph containing the existing vertices and edges plus the
- * newly added edge
+ * @param source
+ * the source vertex of the edge
+ * @param target
+ * the target vertex of the edge
+ * @param edgeValue
+ * the edge value
+ * @return
+ * the new graph containing the existing vertices and edges plus the newly added edge
*/
def addEdge(source: Vertex[K, VV], target: Vertex[K, VV], edgeValue: EV) = {
wrapGraph(jgraph.addEdge(source, target, edgeValue))
@@ -983,9 +1083,11 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes the given vertex and its edges from the graph.
*
- * @param vertex the vertex to remove
- * @return the new graph containing the existing vertices and edges without
- * the removed vertex and its edges
+ * @param vertex
+ * the vertex to remove
+ * @return
+ * the new graph containing the existing vertices and edges without the removed vertex and its
+ * edges
*/
def removeVertex(vertex: Vertex[K, VV]) = {
wrapGraph(jgraph.removeVertex(vertex))
@@ -994,9 +1096,11 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes the given vertex and its edges from the graph.
*
- * @param vertices list of vertices to remove
- * @return the new graph containing the existing vertices and edges without
- * the removed vertex and its edges
+ * @param vertices
+ * list of vertices to remove
+ * @return
+ * the new graph containing the existing vertices and edges without the removed vertex and its
+ * edges
*/
def removeVertices(vertices: List[Vertex[K, VV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.removeVertices(vertices.asJava))
@@ -1005,9 +1109,10 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes all edges that match the given edge from the graph.
*
- * @param edge the edge to remove
- * @return the new graph containing the existing vertices and edges without
- * the removed edges
+ * @param edge
+ * the edge to remove
+ * @return
+ * the new graph containing the existing vertices and edges without the removed edges
*/
def removeEdge(edge: Edge[K, EV]) = {
wrapGraph(jgraph.removeEdge(edge))
@@ -1016,31 +1121,36 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Removes all the edges that match the edges in the given data set from the graph.
*
- * @param edges the list of edges to be removed
- * @return a new graph where the edges have been removed and in which the vertices remained intact
+ * @param edges
+ * the list of edges to be removed
+ * @return
+ * a new graph where the edges have been removed and in which the vertices remained intact
*/
def removeEdges(edges: List[Edge[K, EV]]): Graph[K, VV, EV] = {
wrapGraph(jgraph.removeEdges(edges.asJava))
}
/**
- * Performs union on the vertices and edges sets of the input graphs
- * removing duplicate vertices but maintaining duplicate edges.
+ * Performs union on the vertices and edges sets of the input graphs removing duplicate vertices
+ * but maintaining duplicate edges.
*
- * @param graph the graph to perform union with
- * @return a new graph
+ * @param graph
+ * the graph to perform union with
+ * @return
+ * a new graph
*/
def union(graph: Graph[K, VV, EV]) = {
wrapGraph(jgraph.union(graph.getWrappedGraph))
}
/**
- * Performs Difference on the vertex and edge sets of the input graphs
- * removes common vertices and edges. If a source/target vertex is removed,
- * its corresponding edge will also be removed
+ * Performs Difference on the vertex and edge sets of the input graphs removes common vertices and
+ * edges. If a source/target vertex is removed, its corresponding edge will also be removed
*
- * @param graph the graph to perform difference with
- * @return a new graph where the common vertices and edges have been removed
+ * @param graph
+ * the graph to perform difference with
+ * @return
+ * a new graph where the common vertices and edges have been removed
*/
def difference(graph: Graph[K, VV, EV]) = {
wrapGraph(jgraph.difference(graph.getWrappedGraph))
@@ -1048,196 +1158,240 @@ TypeInformation : ClassTag](jgraph: jg.Graph[K, VV, EV]) {
/**
* Performs intersect on the edge sets of the input graphs. Edges are considered equal, if they
- * have the same source identifier, target identifier and edge value.
- * <p>
- * The method computes pairs of equal edges from the input graphs. If the same edge occurs
- * multiple times in the input graphs, there will be multiple edge pairs to be considered. Each
- * edge instance can only be part of one pair. If the given parameter `distinctEdges` is set
- * to `true`, there will be exactly one edge in the output graph representing all pairs of
- * equal edges. If the parameter is set to `false`, both edges of each pair will be in the
- * output.
- * <p>
- * Vertices in the output graph will have no vertex values.
+ * have the same source identifier, target identifier and edge value. <p> The method computes
+ * pairs of equal edges from the input graphs. If the same edge occurs multiple times in the input
+ * graphs, there will be multiple edge pairs to be considered. Each edge instance can only be part
+ * of one pair. If the given parameter `distinctEdges` is set to `true`, there will be exactly one
+ * edge in the output graph representing all pairs of equal edges. If the parameter is set to
+ * `false`, both edges of each pair will be in the output. <p> Vertices in the output graph will
+ * have no vertex values.
*
- * @param graph the graph to perform intersect with
- * @param distinctEdges if set to { @code true}, there will be exactly one edge in the output
- * graph representing all pairs of equal edges, otherwise, for each pair,
- * both edges will be in the output graph
- * @return a new graph which contains only common vertices and edges from the input graphs
+ * @param graph
+ * the graph to perform intersect with
+ * @param distinctEdges
+ * if set to { @code true}, there will be exactly one edge in the output graph representing all
+ * pairs of equal edges, otherwise, for each pair, both edges will be in the output graph
+ * @return
+ * a new graph which contains only common vertices and edges from the input graphs
*/
def intersect(graph: Graph[K, VV, EV], distinctEdges: Boolean): Graph[K, NullValue, EV] = {
wrapGraph(jgraph.intersect(graph.getWrappedGraph, distinctEdges))
}
/**
- * Compute a reduce transformation over the neighbors' vertex values of each vertex.
- * For each vertex, the transformation consecutively calls a
- * [[ReduceNeighborsFunction]] until only a single value for each vertex remains.
- * The [[ReduceNeighborsFunction]] combines a pair of neighbor vertex values
- * into one new value of the same type.
- *
- * @param reduceNeighborsFunction the reduce function to apply to the neighbors of each vertex.
- * @param direction the edge direction (in-, out-, all-)
- * @return a Dataset of Tuple2, with one tuple per vertex.
- * The first field of the Tuple2 is the vertex ID and the second field
- * is the aggregate value computed by the provided [[ReduceNeighborsFunction]].
- */
- def reduceOnNeighbors(reduceNeighborsFunction: ReduceNeighborsFunction[VV], direction:
- EdgeDirection): DataSet[(K, VV)] = {
+ * Compute a reduce transformation over the neighbors' vertex values of each vertex. For each
+ * vertex, the transformation consecutively calls a [[ReduceNeighborsFunction]] until only a
+ * single value for each vertex remains. The [[ReduceNeighborsFunction]] combines a pair of
+ * neighbor vertex values into one new value of the same type.
+ *
+ * @param reduceNeighborsFunction
+ * the reduce function to apply to the neighbors of each vertex.
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @return
+ * a Dataset of Tuple2, with one tuple per vertex. The first field of the Tuple2 is the vertex
+ * ID and the second field is the aggregate value computed by the provided
+ * [[ReduceNeighborsFunction]].
+ */
+ def reduceOnNeighbors(
+ reduceNeighborsFunction: ReduceNeighborsFunction[VV],
+ direction: EdgeDirection): DataSet[(K, VV)] = {
wrap(jgraph.reduceOnNeighbors(reduceNeighborsFunction, direction)).map(
(i: jtuple.Tuple2[K, VV]) => (i.f0, i.f1))
}
/**
- * Compute a reduce transformation over the neighbors' vertex values of each vertex.
- * For each vertex, the transformation consecutively calls a
- * [[ReduceNeighborsFunction]] until only a single value for each vertex remains.
- * The [[ReduceNeighborsFunction]] combines a pair of neighbor vertex values
- * into one new value of the same type.
- *
- * @param reduceEdgesFunction the reduce function to apply to the edges of each vertex.
- * @param direction the edge direction (in-, out-, all-)
- * @return a Dataset of Tuple2, with one tuple per vertex.
- * The first field of the Tuple2 is the vertex ID and the second field
- * is the aggregate value computed by the provided [[ReduceNeighborsFunction]].
- */
- def reduceOnEdges(reduceEdgesFunction: ReduceEdgesFunction[EV], direction: EdgeDirection):
- DataSet[(K, EV)] = {
+ * Compute a reduce transformation over the neighbors' vertex values of each vertex. For each
+ * vertex, the transformation consecutively calls a [[ReduceNeighborsFunction]] until only a
+ * single value for each vertex remains. The [[ReduceNeighborsFunction]] combines a pair of
+ * neighbor vertex values into one new value of the same type.
+ *
+ * @param reduceEdgesFunction
+ * the reduce function to apply to the edges of each vertex.
+ * @param direction
+ * the edge direction (in-, out-, all-)
+ * @return
+ * a Dataset of Tuple2, with one tuple per vertex. The first field of the Tuple2 is the vertex
+ * ID and the second field is the aggregate value computed by the provided
+ * [[ReduceNeighborsFunction]].
+ */
+ def reduceOnEdges(
+ reduceEdgesFunction: ReduceEdgesFunction[EV],
+ direction: EdgeDirection): DataSet[(K, EV)] = {
wrap(jgraph.reduceOnEdges(reduceEdgesFunction, direction)).map(
(i: jtuple.Tuple2[K, EV]) => (i.f0, i.f1))
}
/**
- * @param algorithm the algorithm to run on the Graph
- * @return the result of the graph algorithm
+ * @param algorithm
+ * the algorithm to run on the Graph
+ * @return
+ * the result of the graph algorithm
*/
- def run[T: TypeInformation : ClassTag](algorithm: GraphAlgorithm[K, VV, EV, T]):
- T = {
+ def run[T: TypeInformation: ClassTag](algorithm: GraphAlgorithm[K, VV, EV, T]): T = {
jgraph.run(algorithm)
}
/**
- * A GraphAnalytic is similar to a GraphAlgorithm but is terminal and results
- * are retrieved via accumulators. A Flink program has a single point of
- * execution. A GraphAnalytic defers execution to the user to allow composing
- * multiple analytics and algorithms into a single program.
+ * A GraphAnalytic is similar to a GraphAlgorithm but is terminal and results are retrieved via
+ * accumulators. A Flink program has a single point of execution. A GraphAnalytic defers execution
+ * to the user to allow composing multiple analytics and algorithms into a single program.
*
- * @param analytic the analytic to run on the Graph
+ * @param analytic
+ * the analytic to run on the Graph
*/
- def run[T: TypeInformation : ClassTag](analytic: GraphAnalytic[K, VV, EV, T]):
- GraphAnalytic[K, VV, EV, T] = {
+ def run[T: TypeInformation: ClassTag](
+ analytic: GraphAnalytic[K, VV, EV, T]): GraphAnalytic[K, VV, EV, T] = {
jgraph.run(analytic)
analytic
}
/**
- * Runs a scatter-gather iteration on the graph.
- * No configuration options are provided.
+ * Runs a scatter-gather iteration on the graph. No configuration options are provided.
*
- * @param scatterFunction the scatter function
- * @param gatherFunction the gather function
- * @param maxIterations maximum number of iterations to perform
- * @return the updated Graph after the scatter-gather iteration has converged or
- * after maximumNumberOfIterations.
+ * @param scatterFunction
+ * the scatter function
+ * @param gatherFunction
+ * the gather function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @return
+ * the updated Graph after the scatter-gather iteration has converged or after
+ * maximumNumberOfIterations.
*/
- def runScatterGatherIteration[M](scatterFunction: ScatterFunction[K, VV, M, EV],
- gatherFunction: SpargelGatherFunction[K, VV, M],
- maxIterations: Int): Graph[K, VV, EV] = {
- wrapGraph(jgraph.runScatterGatherIteration(scatterFunction, gatherFunction,
- maxIterations))
+ def runScatterGatherIteration[M](
+ scatterFunction: ScatterFunction[K, VV, M, EV],
+ gatherFunction: SpargelGatherFunction[K, VV, M],
+ maxIterations: Int): Graph[K, VV, EV] = {
+ wrapGraph(jgraph.runScatterGatherIteration(scatterFunction, gatherFunction, maxIterations))
}
/**
* Runs a scatter-gather iteration on the graph with configuration options.
*
- * @param scatterFunction the scatter function
- * @param gatherFunction the gather function
- * @param maxIterations maximum number of iterations to perform
- * @param parameters the iteration configuration parameters
- * @return the updated Graph after the scatter-gather iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runScatterGatherIteration[M](scatterFunction: ScatterFunction[K, VV, M, EV],
- gatherFunction: SpargelGatherFunction[K, VV, M],
- maxIterations: Int, parameters: ScatterGatherConfiguration):
- Graph[K, VV, EV] = {
- wrapGraph(jgraph.runScatterGatherIteration(scatterFunction, gatherFunction,
- maxIterations, parameters))
- }
-
- /**
- * Runs a Gather-Sum-Apply iteration on the graph.
- * No configuration options are provided.
+ * @param scatterFunction
+ * the scatter function
+ * @param gatherFunction
+ * the gather function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @param parameters
+ * the iteration configuration parameters
+ * @return
+ * the updated Graph after the scatter-gather iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runScatterGatherIteration[M](
+ scatterFunction: ScatterFunction[K, VV, M, EV],
+ gatherFunction: SpargelGatherFunction[K, VV, M],
+ maxIterations: Int,
+ parameters: ScatterGatherConfiguration): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runScatterGatherIteration(scatterFunction, gatherFunction, maxIterations, parameters))
+ }
+
+ /**
+ * Runs a Gather-Sum-Apply iteration on the graph. No configuration options are provided.
*
- * @param gatherFunction the gather function collects information about adjacent
- * vertices and edges
- * @param sumFunction the sum function aggregates the gathered information
- * @param applyFunction the apply function updates the vertex values with the aggregates
- * @param maxIterations maximum number of iterations to perform
- * @tparam M the intermediate type used between gather, sum and apply
- * @return the updated Graph after the gather-sum-apply iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runGatherSumApplyIteration[M](gatherFunction: GSAGatherFunction[VV, EV, M], sumFunction:
- SumFunction[VV, EV, M], applyFunction: ApplyFunction[K, VV, M], maxIterations: Int): Graph[K,
- VV, EV] = {
- wrapGraph(jgraph.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction,
- maxIterations))
+ * @param gatherFunction
+ * the gather function collects information about adjacent vertices and edges
+ * @param sumFunction
+ * the sum function aggregates the gathered information
+ * @param applyFunction
+ * the apply function updates the vertex values with the aggregates
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @tparam M
+ * the intermediate type used between gather, sum and apply
+ * @return
+ * the updated Graph after the gather-sum-apply iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runGatherSumApplyIteration[M](
+ gatherFunction: GSAGatherFunction[VV, EV, M],
+ sumFunction: SumFunction[VV, EV, M],
+ applyFunction: ApplyFunction[K, VV, M],
+ maxIterations: Int): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction, maxIterations))
}
/**
* Runs a Gather-Sum-Apply iteration on the graph with configuration options.
*
- * @param gatherFunction the gather function collects information about adjacent
- * vertices and edges
- * @param sumFunction the sum function aggregates the gathered information
- * @param applyFunction the apply function updates the vertex values with the aggregates
- * @param maxIterations maximum number of iterations to perform
- * @param parameters the iteration configuration parameters
- * @tparam M the intermediate type used between gather, sum and apply
- * @return the updated Graph after the gather-sum-apply iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runGatherSumApplyIteration[M](gatherFunction: GSAGatherFunction[VV, EV, M], sumFunction:
- SumFunction[VV, EV, M], applyFunction: ApplyFunction[K, VV, M], maxIterations: Int,
- parameters: GSAConfiguration): Graph[K, VV, EV] = {
- wrapGraph(jgraph.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction,
- maxIterations, parameters))
- }
-
- /**
- * Runs a vertex-centric iteration on the graph.
- * No configuration options are provided.
+ * @param gatherFunction
+ * the gather function collects information about adjacent vertices and edges
+ * @param sumFunction
+ * the sum function aggregates the gathered information
+ * @param applyFunction
+ * the apply function updates the vertex values with the aggregates
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @param parameters
+ * the iteration configuration parameters
+ * @tparam M
+ * the intermediate type used between gather, sum and apply
+ * @return
+ * the updated Graph after the gather-sum-apply iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runGatherSumApplyIteration[M](
+ gatherFunction: GSAGatherFunction[VV, EV, M],
+ sumFunction: SumFunction[VV, EV, M],
+ applyFunction: ApplyFunction[K, VV, M],
+ maxIterations: Int,
+ parameters: GSAConfiguration): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runGatherSumApplyIteration(
+ gatherFunction,
+ sumFunction,
+ applyFunction,
+ maxIterations,
+ parameters))
+ }
+
+ /**
+ * Runs a vertex-centric iteration on the graph. No configuration options are provided.
*
- * @param computeFunction the compute function
- * @param combineFunction the optional message combiner function
- * @param maxIterations maximum number of iterations to perform
- * @return the updated Graph after the vertex-centric iteration has converged or
- * after maximumNumberOfIterations.
+ * @param computeFunction
+ * the compute function
+ * @param combineFunction
+ * the optional message combiner function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @return
+ * the updated Graph after the vertex-centric iteration has converged or after
+ * maximumNumberOfIterations.
*/
- def runVertexCentricIteration[M](computeFunction: ComputeFunction[K, VV, EV, M],
- combineFunction: MessageCombiner[K, M],
- maxIterations: Int): Graph[K, VV, EV] = {
- wrapGraph(jgraph.runVertexCentricIteration(computeFunction, combineFunction,
- maxIterations))
+ def runVertexCentricIteration[M](
+ computeFunction: ComputeFunction[K, VV, EV, M],
+ combineFunction: MessageCombiner[K, M],
+ maxIterations: Int): Graph[K, VV, EV] = {
+ wrapGraph(jgraph.runVertexCentricIteration(computeFunction, combineFunction, maxIterations))
}
/**
* Runs a vertex-centric iteration on the graph with configuration options.
*
- * @param computeFunction the compute function
- * @param combineFunction the optional message combiner function
- * @param maxIterations maximum number of iterations to perform
- * @param parameters the iteration configuration parameters
- * @return the updated Graph after the vertex-centric iteration has converged or
- * after maximumNumberOfIterations.
- */
- def runVertexCentricIteration[M](computeFunction: ComputeFunction[K, VV, EV, M],
- combineFunction: MessageCombiner[K, M],
- maxIterations: Int, parameters: VertexCentricConfiguration):
- Graph[K, VV, EV] = {
- wrapGraph(jgraph.runVertexCentricIteration(computeFunction, combineFunction,
- maxIterations, parameters))
+ * @param computeFunction
+ * the compute function
+ * @param combineFunction
+ * the optional message combiner function
+ * @param maxIterations
+ * maximum number of iterations to perform
+ * @param parameters
+ * the iteration configuration parameters
+ * @return
+ * the updated Graph after the vertex-centric iteration has converged or after
+ * maximumNumberOfIterations.
+ */
+ def runVertexCentricIteration[M](
+ computeFunction: ComputeFunction[K, VV, EV, M],
+ combineFunction: MessageCombiner[K, M],
+ maxIterations: Int,
+ parameters: VertexCentricConfiguration): Graph[K, VV, EV] = {
+ wrapGraph(
+ jgraph.runVertexCentricIteration(computeFunction, combineFunction, maxIterations, parameters))
}
def validate(validator: GraphValidator[K, VV, EV]): Boolean = {
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala
index ca15dabdcca..9ccd18e27b8 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunction.scala
@@ -15,22 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
import org.apache.flink.api.java.tuple.Tuple3
import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.util.Collector
-
-abstract class NeighborsFunction[K, VV, EV, T] extends org.apache.flink.graph
-.NeighborsFunction[K, VV, EV, T] {
+abstract class NeighborsFunction[K, VV, EV, T]
+ extends org.apache.flink.graph.NeighborsFunction[K, VV, EV, T] {
def iterateNeighbors(neighbors: Iterable[(K, Edge[K, EV], Vertex[K, VV])], out: Collector[T])
- override def iterateNeighbors(neighbors: java.lang.Iterable[Tuple3[K, Edge[K, EV], Vertex[K,
- VV]]], out: Collector[T]) = {
- val scalaIterable = scala.collection.JavaConversions.iterableAsScalaIterable(neighbors)
+ override def iterateNeighbors(
+ neighbors: java.lang.Iterable[Tuple3[K, Edge[K, EV], Vertex[K, VV]]],
+ out: Collector[T]) = {
+ val scalaIterable = scala.collection.JavaConversions
+ .iterableAsScalaIterable(neighbors)
.map(jtuple => (jtuple.f0, jtuple.f1, jtuple.f2))
iterateNeighbors(scalaIterable, out)
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala
index cefc277c00a..d9fdd13af10 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/NeighborsFunctionWithVertexValue.scala
@@ -15,25 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala
-import java.lang
-
import org.apache.flink.api.java.tuple.Tuple2
import org.apache.flink.graph.{Edge, Vertex}
import org.apache.flink.util.Collector
+import java.lang
-abstract class NeighborsFunctionWithVertexValue[K, VV, EV, T] extends org.apache.flink.graph
-.NeighborsFunctionWithVertexValue[K, VV, EV, T] {
+abstract class NeighborsFunctionWithVertexValue[K, VV, EV, T]
+ extends org.apache.flink.graph.NeighborsFunctionWithVertexValue[K, VV, EV, T] {
- def iterateNeighbors(vertex: Vertex[K, VV], neighbors: Iterable[(Edge[K, EV], Vertex[K, VV])
- ], out: Collector[T]): Unit
+ def iterateNeighbors(
+ vertex: Vertex[K, VV],
+ neighbors: Iterable[(Edge[K, EV], Vertex[K, VV])],
+ out: Collector[T]): Unit
- override def iterateNeighbors(vertex: Vertex[K, VV], neighbors: lang.Iterable[Tuple2[Edge[K,
- EV], Vertex[K, VV]]], out: Collector[T]): Unit = {
- val scalaIterable = scala.collection.JavaConversions.iterableAsScalaIterable(neighbors)
+ override def iterateNeighbors(
+ vertex: Vertex[K, VV],
+ neighbors: lang.Iterable[Tuple2[Edge[K, EV], Vertex[K, VV]]],
+ out: Collector[T]): Unit = {
+ val scalaIterable = scala.collection.JavaConversions
+ .iterableAsScalaIterable(neighbors)
.map(jtuple => (jtuple.f0, jtuple.f1))
iterateNeighbors(vertex, scalaIterable, out)
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala
index 83b6b6d22dc..24a5726d31a 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/package.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph
import org.apache.flink.api.common.typeinfo.TypeInformation
@@ -23,14 +22,11 @@ import org.apache.flink.graph.{Graph => JGraph}
import _root_.scala.reflect.ClassTag
-
package object scala {
private[flink] def wrapGraph[
- K: TypeInformation : ClassTag,
- VV: TypeInformation : ClassTag,
- EV: TypeInformation : ClassTag](
- javagraph: JGraph[K, VV, EV])
- : scala.Graph[K, VV, EV] = {
+ K: TypeInformation: ClassTag,
+ VV: TypeInformation: ClassTag,
+ EV: TypeInformation: ClassTag](javagraph: JGraph[K, VV, EV]): scala.Graph[K, VV, EV] = {
new scala.Graph[K, VV, EV](javagraph)
}
}
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala
index 909dbb49aaf..9269087b4e3 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/EdgeToTuple3Map.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.utils
import org.apache.flink.api.common.functions.MapFunction
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala
index fd6b8c535bd..4e4627e35c9 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple2ToVertexMap.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.utils
import org.apache.flink.api.common.functions.MapFunction
diff --git a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple3ToEdgeMap.scala b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple3ToEdgeMap.scala
index d0e07cc8c58..faa58f5e08d 100644
--- a/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple3ToEdgeMap.scala
+++ b/flink-libraries/flink-gelly-scala/src/main/scala/org/apache/flink/graph/scala/utils/Tuple3ToEdgeMap.scala
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.graph.scala.utils
... 197714 lines suppressed ...