You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by wy...@apache.org on 2019/03/27 08:15:47 UTC
[incubator-nemo] branch master updated: [NEMO-361] Consistency on
indentations (#207)
This is an automated email from the ASF dual-hosted git repository.
wylee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new 55e05f4 [NEMO-361] Consistency on indentations (#207)
55e05f4 is described below
commit 55e05f468bddc7b7e1fdba25ddccce7d3df330d3
Author: Won Wook SONG <wo...@apache.org>
AuthorDate: Wed Mar 27 17:15:43 2019 +0900
[NEMO-361] Consistency on indentations (#207)
JIRA: [NEMO-361: Consistency on indentations](https://issues.apache.org/jira/projects/NEMO/issues/NEMO-361)
**Major changes:**
- Runs the 'Reformat code' command on IntelliJ to auto-indent files.
- Re-organizes imports in an alphabetical order
**Minor changes to note:**
- None
**Tests for the changes:**
- None, it simply fixes the indentations and imports
**Other comments:**
- None
Closes #207
---
bin/json2dot.py | 566 ++++++++--------
bin/metric-parser.py | 125 ++--
checkstyle.xml | 270 ++++----
client/pom.xml | 101 +--
.../org/apache/nemo/client/ClientEndpoint.java | 5 +-
.../org/apache/nemo/client/DriverEndpoint.java | 8 +-
.../org/apache/nemo/client/DriverRPCServer.java | 15 +-
.../java/org/apache/nemo/client/JobLauncher.java | 86 +--
.../nemo/client/beam/NemoPipelineResult.java | 2 +-
.../org/apache/nemo/client/beam/NemoRunner.java | 7 +-
.../nemo/client/beam/NemoRunnerRegistrar.java | 2 +-
.../apache/reef/runtime/yarn/ClassPathBuilder.java | 4 +-
.../apache/nemo/client/ClientDriverRPCTest.java | 24 +-
.../org/apache/nemo/client/ClientEndpointTest.java | 10 +-
common/pom.xml | 67 +-
.../nemo/common/DirectByteArrayOutputStream.java | 2 +
.../java/org/apache/nemo/common/HashRange.java | 5 +-
.../main/java/org/apache/nemo/common/KeyRange.java | 1 +
.../src/main/java/org/apache/nemo/common/Pair.java | 20 +-
.../java/org/apache/nemo/common/StateMachine.java | 32 +-
.../src/main/java/org/apache/nemo/common/Util.java | 12 +-
.../nemo/common/coder/BytesDecoderFactory.java | 3 +-
.../nemo/common/coder/BytesEncoderFactory.java | 4 +-
.../nemo/common/coder/IntDecoderFactory.java | 7 +-
.../nemo/common/coder/IntEncoderFactory.java | 5 +-
.../nemo/common/coder/LongDecoderFactory.java | 3 +-
.../nemo/common/coder/LongEncoderFactory.java | 1 +
.../nemo/common/coder/PairDecoderFactory.java | 8 +-
.../nemo/common/coder/PairEncoderFactory.java | 9 +-
.../main/java/org/apache/nemo/common/dag/DAG.java | 43 +-
.../org/apache/nemo/common/dag/DAGBuilder.java | 39 +-
.../org/apache/nemo/common/dag/DAGInterface.java | 36 +-
.../main/java/org/apache/nemo/common/dag/Edge.java | 7 +-
.../java/org/apache/nemo/common/dag/Vertex.java | 1 +
.../common/eventhandler/CommonEventHandler.java | 2 +
.../common/eventhandler/CompilerEventHandler.java | 1 +
.../common/eventhandler/RuntimeEventHandler.java | 1 +
.../nemo/common/exception/ContainerException.java | 1 +
.../exception/IllegalEdgeOperationException.java | 1 +
.../common/exception/IllegalMessageException.java | 1 +
.../exception/IllegalStateTransitionException.java | 1 +
.../exception/IllegalVertexOperationException.java | 1 +
.../exception/InvalidParameterException.java | 1 +
.../nemo/common/exception/JsonParseException.java | 1 +
.../nemo/common/exception/MetricException.java | 2 +
.../common/exception/NodeConnectionException.java | 1 +
.../exception/PhysicalPlanGenerationException.java | 2 +
.../nemo/common/exception/SchedulingException.java | 1 +
.../exception/UnknownExecutionStateException.java | 1 +
.../exception/UnknownFailureCauseException.java | 1 +
.../exception/UnrecoverableFailureException.java | 1 +
.../exception/UnsupportedBlockStoreException.java | 1 +
.../exception/UnsupportedCommPatternException.java | 1 +
.../UnsupportedExecutionPropertyException.java | 1 +
.../exception/UnsupportedMethodException.java | 1 +
.../exception/UnsupportedMetricException.java | 1 +
.../exception/UnsupportedPartitionerException.java | 1 +
.../nemo/common/ir/BoundedIteratorReadable.java | 2 +
.../main/java/org/apache/nemo/common/ir/IRDAG.java | 61 +-
.../java/org/apache/nemo/common/ir/IdManager.java | 4 +-
.../org/apache/nemo/common/ir/OutputCollector.java | 8 +-
.../java/org/apache/nemo/common/ir/Readable.java | 6 +-
.../org/apache/nemo/common/ir/edge/IREdge.java | 17 +-
.../AdditionalOutputTagProperty.java | 2 +
.../CommunicationPatternProperty.java | 5 +-
.../edge/executionproperty/DataFlowProperty.java | 2 +
.../executionproperty/DataPersistenceProperty.java | 2 +
.../edge/executionproperty/DataStoreProperty.java | 2 +
.../DuplicateEdgeGroupProperty.java | 2 +
.../executionproperty/MessageIdEdgeProperty.java | 2 +
.../executionproperty/PartitionSetProperty.java | 6 +-
.../executionproperty/PartitionerProperty.java | 6 +-
.../executionproperty/EdgeExecutionProperty.java | 2 +
.../ir/executionproperty/ExecutionProperty.java | 2 +
.../ir/executionproperty/ExecutionPropertyMap.java | 39 +-
.../executionproperty/VertexExecutionProperty.java | 2 +
.../nemo/common/ir/vertex/CachedSourceVertex.java | 1 +
.../org/apache/nemo/common/ir/vertex/IRVertex.java | 10 +-
.../common/ir/vertex/InMemorySourceVertex.java | 5 +-
.../apache/nemo/common/ir/vertex/LoopVertex.java | 36 +-
.../nemo/common/ir/vertex/OperatorVertex.java | 2 +
.../apache/nemo/common/ir/vertex/SourceVertex.java | 2 +
.../ClonedSchedulingProperty.java | 7 +-
.../executionproperty/MessageIdVertexProperty.java | 2 +
.../executionproperty/ParallelismProperty.java | 8 +-
.../ResourcePriorityProperty.java | 2 +
.../executionproperty/ResourceSiteProperty.java | 32 +-
.../executionproperty/ResourceSlotProperty.java | 2 +-
.../executionproperty/ScheduleGroupProperty.java | 2 +
.../transform/MessageAggregatorTransform.java | 1 +
.../vertex/transform/MessageBarrierTransform.java | 2 +
.../vertex/transform/NoWatermarkEmitTransform.java | 1 +
.../ir/vertex/transform/StreamTransform.java | 1 +
.../nemo/common/ir/vertex/transform/Transform.java | 8 +-
.../ir/vertex/utility/MessageAggregatorVertex.java | 1 +
.../ir/vertex/utility/MessageBarrierVertex.java | 1 +
.../common/ir/vertex/utility/SamplingVertex.java | 4 +-
.../nemo/common/partitioner/HashPartitioner.java | 2 +-
.../nemo/common/partitioner/Partitioner.java | 2 +-
.../java/org/apache/nemo/common/pass/Pass.java | 3 +
.../apache/nemo/common/test/EmptyComponents.java | 6 +-
.../apache/nemo/common/test/ExampleTestUtil.java | 99 +--
.../test/java/org/apache/nemo/common/DAGTest.java | 5 +-
.../test/java/org/apache/nemo/common/PairTest.java | 1 -
.../org/apache/nemo/common/StateMachineTest.java | 1 -
.../apache/nemo/common/coder/CoderFactoryTest.java | 4 +-
.../java/org/apache/nemo/common/ir/IRDAGTest.java | 46 +-
.../org/apache/nemo/common/ir/LoopVertexTest.java | 28 +-
.../ExecutionPropertyMapTest.java | 4 +-
.../java/org/apache/nemo/common/util/UtilTest.java | 28 +-
compiler/backend/pom.xml | 55 +-
.../org/apache/nemo/compiler/backend/Backend.java | 1 +
.../nemo/compiler/backend/nemo/NemoBackend.java | 1 +
.../compiler/backend/nemo/NemoPlanRewriter.java | 11 +-
.../compiler/backend/nemo/NemoBackendTest.java | 12 +-
compiler/frontend/beam/pom.xml | 3 +-
.../compiler/frontend/beam/BeamKeyExtractor.java | 2 +-
.../frontend/beam/InMemorySideInputReader.java | 9 +-
.../frontend/beam/PipelineTranslationContext.java | 30 +-
.../compiler/frontend/beam/PipelineTranslator.java | 77 ++-
.../compiler/frontend/beam/PipelineVisitor.java | 2 +-
.../compiler/frontend/beam/SideInputElement.java | 1 +
.../frontend/beam/coder/BeamDecoderFactory.java | 6 +-
.../frontend/beam/coder/BeamEncoderFactory.java | 3 +-
.../frontend/beam/coder/SideInputCoder.java | 2 +
.../beam/source/BeamBoundedSourceVertex.java | 7 +-
.../beam/source/BeamUnboundedSourceVertex.java | 9 +-
.../beam/transform/AbstractDoFnTransform.java | 35 +-
.../beam/transform/CombineFnFinalTransform.java | 2 +
.../beam/transform/CombineFnPartialTransform.java | 2 +
.../beam/transform/CreateViewTransform.java | 12 +-
.../beam/transform/DefaultOutputManager.java | 3 +-
.../frontend/beam/transform/DoFnTransform.java | 17 +-
.../frontend/beam/transform/FlattenTransform.java | 1 +
.../GroupByKeyAndWindowDoFnTransform.java | 35 +-
.../beam/transform/GroupByKeyTransform.java | 3 +-
.../beam/transform/LoopCompositeTransform.java | 5 +-
.../beam/transform/PushBackDoFnTransform.java | 19 +-
.../beam/transform/SideInputTransform.java | 2 +
.../frontend/beam/transform/WindowFnTransform.java | 6 +-
compiler/frontend/spark/pom.xml | 227 +++----
.../frontend/spark/coder/SparkDecoderFactory.java | 2 +
.../frontend/spark/coder/SparkEncoderFactory.java | 2 +
.../frontend/spark/core/JavaSparkContext.java | 7 +-
.../frontend/spark/core/SparkBroadcast.java | 3 +-
.../compiler/frontend/spark/core/SparkContext.java | 6 +-
.../frontend/spark/core/SparkFrontendUtils.java | 23 +-
.../frontend/spark/core/rdd/JavaPairRDD.java | 3 +-
.../compiler/frontend/spark/core/rdd/JavaRDD.java | 17 +-
.../source/SparkDatasetBoundedSourceVertex.java | 12 +-
.../source/SparkTextFileBoundedSourceVertex.java | 19 +-
.../frontend/spark/sql/DataFrameReader.java | 5 +-
.../nemo/compiler/frontend/spark/sql/Dataset.java | 27 +-
.../spark/sql/NemoSparkUserFacingClass.java | 2 +
.../compiler/frontend/spark/sql/SparkSession.java | 8 +-
.../frontend/spark/transform/CollectTransform.java | 3 +-
.../frontend/spark/transform/FlatMapTransform.java | 2 +
.../spark/transform/GroupByKeyTransform.java | 8 +-
.../spark/transform/HDFSTextFileTransform.java | 11 +-
.../spark/transform/LocalTextFileTransform.java | 5 +-
.../spark/transform/MapToPairTransform.java | 2 +
.../frontend/spark/transform/MapTransform.java | 12 +-
.../spark/transform/ReduceByKeyTransform.java | 7 +-
.../frontend/spark/transform/ReduceTransform.java | 6 +-
compiler/optimizer/pom.xml | 75 +--
.../nemo/compiler/optimizer/NemoOptimizer.java | 2 +-
.../apache/nemo/compiler/optimizer/Optimizer.java | 2 +-
.../MapReduceDisaggregationOptimization.java | 1 +
.../pass/compiletime/CompileTimePass.java | 1 +
.../compiletime/annotating/AnnotatingPass.java | 4 +-
.../compiletime/annotating/CompressionPass.java | 1 +
.../annotating/DefaultDataPersistencePass.java | 22 +-
.../annotating/DefaultDataStorePass.java | 6 +-
.../annotating/DefaultEdgeDecoderPass.java | 12 +-
.../annotating/DefaultEdgeEncoderPass.java | 12 +-
.../annotating/DefaultParallelismPass.java | 37 +-
.../annotating/DefaultScheduleGroupPass.java | 31 +-
.../annotating/DuplicateEdgeGroupSizePass.java | 36 +-
.../annotating/LargeShuffleAnnotatingPass.java | 13 +-
.../annotating/PipeTransferForAllEdgesPass.java | 4 +-
.../annotating/ResourceLocalityPass.java | 4 +-
.../compiletime/annotating/ResourceSitePass.java | 29 +-
.../compiletime/annotating/ResourceSlotPass.java | 4 +-
.../annotating/ShuffleEdgePushPass.java | 2 +-
.../annotating/TransientResourceDataStorePass.java | 14 +-
.../annotating/TransientResourcePriorityPass.java | 14 +-
.../compiletime/annotating/UpfrontCloningPass.java | 22 +-
.../pass/compiletime/composite/CompositePass.java | 5 +-
.../composite/DefaultCompositePass.java | 18 +-
.../composite/LargeShuffleCompositePass.java | 6 +-
.../composite/LoopOptimizationCompositePass.java | 10 +-
.../composite/TransientResourceCompositePass.java | 6 +-
.../CommonSubexpressionEliminationPass.java | 11 +-
.../compiletime/reshaping/LoopExtractionPass.java | 37 +-
.../compiletime/reshaping/LoopOptimizations.java | 16 +-
.../compiletime/reshaping/LoopUnrollingPass.java | 1 +
.../pass/compiletime/reshaping/ReshapingPass.java | 3 +-
.../reshaping/SamplingSkewReshapingPass.java | 19 +-
.../compiletime/reshaping/SkewHandlingUtil.java | 5 +-
.../compiletime/reshaping/SkewReshapingPass.java | 11 +-
.../optimizer/pass/runtime/RunTimePass.java | 1 +
.../optimizer/pass/runtime/SkewRunTimePass.java | 8 +-
.../compiler/optimizer/policy/BasicPullPolicy.java | 6 +-
.../policy/ConditionalLargeShufflePolicy.java | 13 +-
.../compiler/optimizer/policy/DataSkewPolicy.java | 10 +-
.../compiler/optimizer/policy/DefaultPolicy.java | 4 +-
.../policy/DefaultPolicyWithSeparatePass.java | 10 +-
.../optimizer/policy/DisaggregationPolicy.java | 8 +-
.../optimizer/policy/LargeShufflePolicy.java | 8 +-
.../nemo/compiler/optimizer/policy/Policy.java | 2 +-
.../compiler/optimizer/policy/PolicyBuilder.java | 17 +-
.../nemo/compiler/optimizer/policy/PolicyImpl.java | 14 +-
.../policy/SamplingLargeShuffleSkewPolicy.java | 4 +-
.../optimizer/policy/TransientResourcePolicy.java | 8 +-
.../optimizer/policy/PolicyBuilderTest.java | 6 +-
compiler/pom.xml | 3 +-
compiler/test/pom.xml | 129 ++--
.../org/apache/nemo/compiler/CompilerTestUtil.java | 31 +-
.../nemo/compiler/optimizer/policy/TestPolicy.java | 7 +-
.../compiler/backend/nemo/DAGConverterTest.java | 6 +-
.../frontend/beam/BeamFrontendALSTest.java | 4 +-
.../frontend/beam/BeamFrontendMLRTest.java | 2 -
.../beam/transform/CreateViewTransformTest.java | 3 +-
.../frontend/beam/transform/DoFnTransformTest.java | 6 +-
.../GroupByKeyAndWindowDoFnTransformTest.java | 8 +-
.../beam/transform/TestOutputCollector.java | 9 +-
.../annotating/DefaultEdgeCoderPassTest.java | 2 -
.../annotating/DefaultParallelismPassTest.java | 8 +-
.../annotating/DefaultScheduleGroupPassTest.java | 44 +-
.../composite/DisaggregationPassTest.java | 12 +-
.../composite/LargeShuffleCompositePassTest.java | 32 +-
.../composite/SkewCompositePassTest.java | 21 +-
.../TransientResourceCompositePassTest.java | 2 -
.../CommonSubexpressionEliminationPassTest.java | 25 +-
.../reshaping/LoopExtractionPassTest.java | 3 -
.../LoopInvariantCodeMotionALSInefficientTest.java | 5 +-
.../reshaping/LoopInvariantCodeMotionPassTest.java | 9 +-
.../reshaping/LoopUnrollingPassTest.java | 12 +-
.../pass/runtime/SkewRuntimePassTest.java | 3 +-
conf/pom.xml | 35 +-
.../main/java/org/apache/nemo/conf/JobConf.java | 36 +-
examples/beam/pom.xml | 191 +++---
.../nemo/examples/beam/AlternatingLeastSquare.java | 54 +-
.../beam/AlternatingLeastSquareInefficient.java | 67 +-
.../org/apache/nemo/examples/beam/Broadcast.java | 25 +-
.../nemo/examples/beam/GenericSourceSink.java | 47 +-
.../nemo/examples/beam/MinimalWordCount.java | 4 +
.../beam/MultinomialLogisticRegression.java | 85 +--
.../examples/beam/NemoPipelineOptionsFactory.java | 1 +
.../nemo/examples/beam/NetworkTraceAnalysis.java | 74 ++-
.../nemo/examples/beam/PartitionWordsByLength.java | 46 +-
.../apache/nemo/examples/beam/PerKeyMedian.java | 49 +-
.../nemo/examples/beam/PerPercentileAverage.java | 75 ++-
.../nemo/examples/beam/WindowedBroadcast.java | 8 +-
.../nemo/examples/beam/WindowedWordCount.java | 4 +-
.../org/apache/nemo/examples/beam/WordCount.java | 37 +-
.../nemo/examples/beam/WriteOneFilePerWindow.java | 95 +--
.../beam/AlternatingLeastSquareITCase.java | 16 +-
.../nemo/examples/beam/BeamSimpleSumSQLITCase.java | 16 +-
.../apache/nemo/examples/beam/BroadcastITCase.java | 26 +-
.../beam/MultinomialLogisticRegressionITCase.java | 15 +-
.../examples/beam/NetworkTraceAnalysisITCase.java | 39 +-
.../beam/PartitionWordsByLengthITCase.java | 28 +-
.../nemo/examples/beam/PerKeyMedianITCase.java | 22 +-
.../examples/beam/PerPercentileAverageITCase.java | 10 +-
.../examples/beam/WindowedBroadcastITCase.java | 3 +-
.../examples/beam/WindowedWordCountITCase.java | 23 +-
.../apache/nemo/examples/beam/WordCountITCase.java | 72 +--
...iveSpeculativeCloningPolicyParallelismFive.java | 7 +-
.../beam/policy/DataSkewPolicyParallelismFive.java | 4 +-
.../beam/policy/DefaultPolicyParallelismFive.java | 4 +-
.../DisaggregationPolicyParallelismFive.java | 6 +-
.../policy/LargeShufflePolicyParallelismFive.java | 4 +-
.../nemo/examples/beam/policy/PolicyTestUtil.java | 8 +-
.../TransientResourcePolicyParallelismFive.java | 8 +-
.../TransientResourcePolicyParallelismTen.java | 8 +-
.../UpfrontSchedulingPolicyParallelismFive.java | 8 +-
examples/pom.xml | 3 +-
examples/spark/pom.xml | 263 ++++----
.../apache/nemo/examples/spark/JavaMapReduce.java | 29 +-
.../apache/nemo/examples/spark/JavaSparkPi.java | 9 +-
.../nemo/examples/spark/JavaWordAndLineCount.java | 7 +-
.../apache/nemo/examples/spark/JavaWordCount.java | 7 +-
.../examples/spark/sql/JavaSparkSQLExample.java | 86 +--
.../spark/sql/JavaUserDefinedTypedAggregation.java | 24 +-
.../sql/JavaUserDefinedUntypedAggregation.java | 11 +-
.../org/apache/nemo/examples/spark/MRJava.java | 22 +-
.../org/apache/nemo/examples/spark/SparkJava.java | 33 +-
.../org/apache/nemo/examples/spark/SparkScala.java | 32 +-
formatter.xml | 709 ++++++++++++---------
pom.xml | 609 +++++++++---------
runtime/common/pom.xml | 119 ++--
.../NativeChannelImplementationSelector.java | 12 +-
.../common/NettyChannelImplementationSelector.java | 1 +
.../apache/nemo/runtime/common/ReplyFutureMap.java | 6 +-
.../nemo/runtime/common/RuntimeTestUtil.java | 6 +-
.../common/exception/AbsentBlockException.java | 2 +-
.../runtime/common/message/MessageContext.java | 2 +-
.../runtime/common/message/MessageEnvironment.java | 7 +-
.../runtime/common/message/MessageListener.java | 4 +-
.../nemo/runtime/common/message/MessageSender.java | 4 +-
.../message/PersistentConnectionToMasterMap.java | 12 +-
.../common/message/grpc/GrpcMessageClient.java | 33 +-
.../common/message/grpc/GrpcMessageContext.java | 5 +-
.../message/grpc/GrpcMessageEnvironment.java | 14 +-
.../common/message/grpc/GrpcMessageSender.java | 1 +
.../common/message/grpc/GrpcMessageServer.java | 43 +-
.../common/message/local/LocalMessageContext.java | 8 +-
.../message/local/LocalMessageDispatcher.java | 12 +-
.../message/local/LocalMessageEnvironment.java | 12 +-
.../common/message/local/LocalMessageSender.java | 7 +-
.../common/message/ncs/ControlMessageCodec.java | 2 +-
.../common/message/ncs/NcsMessageContext.java | 1 -
.../common/message/ncs/NcsMessageEnvironment.java | 18 +-
.../common/message/ncs/NcsMessageSender.java | 8 +-
.../runtime/common/metric/DataTransferEvent.java | 2 +
.../apache/nemo/runtime/common/metric/Event.java | 5 +-
.../nemo/runtime/common/metric/JobMetric.java | 4 +
.../apache/nemo/runtime/common/metric/Metric.java | 2 +
.../nemo/runtime/common/metric/MetricUtils.java | 18 +-
.../nemo/runtime/common/metric/StateMetric.java | 5 +-
.../common/metric/StateTransitionEvent.java | 3 +
.../nemo/runtime/common/metric/TaskMetric.java | 9 +-
.../runtime/common/plan/PhysicalPlanGenerator.java | 15 +-
.../nemo/runtime/common/plan/PlanRewriter.java | 4 +-
.../nemo/runtime/common/plan/RuntimeEdge.java | 11 +-
.../org/apache/nemo/runtime/common/plan/Stage.java | 6 +-
.../apache/nemo/runtime/common/plan/StageEdge.java | 8 +-
.../nemo/runtime/common/plan/StagePartitioner.java | 23 +-
.../org/apache/nemo/runtime/common/plan/Task.java | 6 +-
.../nemo/runtime/common/state/BlockState.java | 2 +-
.../nemo/runtime/common/state/PlanState.java | 6 +-
.../nemo/runtime/common/state/StageState.java | 6 +-
.../nemo/runtime/common/state/TaskState.java | 2 +-
.../common/message/local/LocalMessageTest.java | 27 +-
.../common/plan/PhysicalPlanGeneratorTest.java | 10 +-
.../runtime/common/plan/StagePartitionerTest.java | 12 +-
runtime/driver/pom.xml | 95 +--
.../java/org/apache/nemo/driver/NemoDriver.java | 52 +-
.../apache/nemo/driver/UserApplicationRunner.java | 4 +-
runtime/executor/pom.xml | 111 ++--
.../org/apache/nemo/runtime/executor/Executor.java | 77 +--
.../nemo/runtime/executor/MetricManagerWorker.java | 43 +-
.../nemo/runtime/executor/MetricMessageSender.java | 5 +-
.../nemo/runtime/executor/TaskStateManager.java | 53 +-
.../runtime/executor/TransformContextImpl.java | 1 +
.../executor/bytetransfer/ByteInputContext.java | 12 +-
.../executor/bytetransfer/ByteOutputContext.java | 25 +-
.../executor/bytetransfer/ByteTransfer.java | 19 +-
.../executor/bytetransfer/ByteTransferContext.java | 24 +-
.../executor/bytetransfer/ByteTransport.java | 79 +--
.../ByteTransportChannelInitializer.java | 18 +-
.../bytetransfer/ClosableBlockingQueue.java | 3 +-
.../executor/bytetransfer/ContextManager.java | 75 ++-
.../executor/bytetransfer/ControlFrameEncoder.java | 16 +-
.../executor/bytetransfer/DataFrameEncoder.java | 11 +-
.../executor/bytetransfer/FrameDecoder.java | 28 +-
.../runtime/executor/data/BlockManagerWorker.java | 149 ++---
.../executor/data/BlockTransferThrottler.java | 2 +
.../executor/data/BroadcastManagerWorker.java | 9 +-
.../nemo/runtime/executor/data/DataUtil.java | 32 +-
.../nemo/runtime/executor/data/FileArea.java | 6 +-
.../nemo/runtime/executor/data/PipeContainer.java | 5 +-
.../runtime/executor/data/PipeManagerWorker.java | 6 +-
.../runtime/executor/data/SerializerManager.java | 8 +-
.../runtime/executor/data/block/FileBlock.java | 25 +-
.../data/block/NonSerializedMemoryBlock.java | 4 +-
.../executor/data/block/SerializedMemoryBlock.java | 4 +-
.../executor/data/metadata/FileMetadata.java | 7 +-
.../executor/data/metadata/LocalFileMetadata.java | 1 +
.../executor/data/metadata/PartitionMetadata.java | 1 +
.../executor/data/metadata/RemoteFileMetadata.java | 15 +-
.../data/partition/NonSerializedPartition.java | 5 +-
.../runtime/executor/data/partition/Partition.java | 2 +
.../data/partition/SerializedPartition.java | 11 +-
.../executor/data/stores/AbstractBlockStore.java | 1 +
.../executor/data/stores/GlusterFileStore.java | 15 +-
.../executor/data/stores/LocalFileStore.java | 16 +-
.../runtime/executor/data/stores/MemoryStore.java | 2 +-
.../data/stores/SerializedMemoryStore.java | 3 +-
.../streamchainer/CompressionStreamChainer.java | 2 +-
.../streamchainer/DecompressionStreamChainer.java | 2 +-
.../executor/data/streamchainer/Serializer.java | 5 +-
.../executor/datatransfer/BlockInputReader.java | 11 +-
.../executor/datatransfer/BlockOutputWriter.java | 21 +-
.../datatransfer/DataFetcherOutputCollector.java | 6 +-
.../runtime/executor/datatransfer/InputReader.java | 2 +-
.../datatransfer/InputWatermarkManager.java | 9 +-
.../datatransfer/MultiInputWatermarkManager.java | 3 +-
.../datatransfer/NemoEventEncoderFactory.java | 1 +
.../datatransfer/NextIntraTaskOperatorInfo.java | 2 +-
.../OperatorVertexOutputCollector.java | 10 +-
.../datatransfer/OperatorWatermarkCollector.java | 1 +
.../executor/datatransfer/OutputWriter.java | 3 +-
.../executor/datatransfer/PipeOutputWriter.java | 9 +-
.../datatransfer/SingleInputWatermarkManager.java | 1 +
.../nemo/runtime/executor/task/DataFetcher.java | 5 +-
.../task/MultiThreadParentTaskDataFetcher.java | 11 +-
.../executor/task/SourceVertexDataFetcher.java | 3 +-
.../nemo/runtime/executor/task/TaskExecutor.java | 105 ++-
.../nemo/runtime/executor/task/VertexHarness.java | 3 +-
.../nemo/runtime/executor/MetricFlushTest.java | 31 +-
.../runtime/executor/TransformContextImplTest.java | 4 +-
.../nemo/runtime/executor/data/BlockStoreTest.java | 230 +++----
.../nemo/runtime/executor/data/BlockTest.java | 4 +-
.../executor/data/BlockTransferThrottlerTest.java | 9 +-
.../executor/datatransfer/DataTransferTest.java | 78 +--
.../datatransfer/InputWatermarkManagerTest.java | 5 +-
.../executor/task/ParentTaskDataFetcherTest.java | 18 +-
.../runtime/executor/task/TaskExecutorTest.java | 192 +++---
runtime/master/pom.xml | 145 ++---
.../nemo/runtime/master/BlockManagerMaster.java | 16 +-
.../apache/nemo/runtime/master/BlockMetadata.java | 2 +-
.../org/apache/nemo/runtime/master/ClientRPC.java | 9 +-
.../nemo/runtime/master/PipeManagerMaster.java | 1 +
.../nemo/runtime/master/PlanStateManager.java | 33 +-
.../apache/nemo/runtime/master/RuntimeMaster.java | 65 +-
.../runtime/master/metric/MetricBroadcaster.java | 9 +-
.../runtime/master/metric/MetricManagerMaster.java | 14 +-
.../master/metric/MetricMessageHandler.java | 5 +-
.../nemo/runtime/master/metric/MetricStore.java | 53 +-
.../runtime/master/resource/ContainerManager.java | 38 +-
.../master/resource/ExecutorRepresenter.java | 40 +-
.../master/resource/ResourceSpecification.java | 2 +-
.../runtime/master/scheduler/BatchScheduler.java | 25 +-
.../ContainerTypeAwareSchedulingConstraint.java | 4 +-
.../runtime/master/scheduler/ExecutorRegistry.java | 18 +-
.../MinOccupancyFirstSchedulingPolicy.java | 11 +-
.../scheduler/NodeShareSchedulingConstraint.java | 4 +-
.../scheduler/PendingTaskCollectionPointer.java | 6 +-
.../nemo/runtime/master/scheduler/Scheduler.java | 2 +-
.../scheduler/SchedulingConstraintRegistry.java | 18 +-
.../runtime/master/scheduler/SchedulingPolicy.java | 4 +-
.../master/scheduler/StreamingScheduler.java | 3 +-
.../runtime/master/scheduler/TaskDispatcher.java | 20 +-
.../runtime/master/servlet/AllMetricServlet.java | 2 +-
.../runtime/master/servlet/JobMetricServlet.java | 4 +-
.../runtime/master/servlet/StageMetricServlet.java | 4 +-
.../runtime/master/servlet/TaskMetricServlet.java | 4 +-
.../master/servlet/WebSocketMetricAdapter.java | 5 +-
.../runtime/master/BlockManagerMasterTest.java | 13 +-
.../nemo/runtime/master/ContainerManagerTest.java | 32 +-
.../nemo/runtime/master/PlanStateManagerTest.java | 14 +-
.../AntiAffinitySchedulingConstraintTest.java | 4 +-
.../master/scheduler/BatchSchedulerTest.java | 35 +-
...ContainerTypeAwareSchedulingConstraintTest.java | 18 +-
.../FreeSlotSchedulingConstraintTest.java | 11 +-
.../LocalitySchedulingConstraintTest.java | 29 +-
.../MinOccupancyFirstSchedulingPolicyTest.java | 10 +-
.../PendingTaskCollectionPointerTest.java | 6 +-
.../master/scheduler/SchedulerTestUtil.java | 18 +-
.../SchedulingConstraintnRegistryTest.java | 16 +-
.../master/scheduler/StreamingSchedulerTest.java | 6 +-
.../runtime/master/scheduler/TaskRetryTest.java | 30 +-
runtime/pom.xml | 3 +-
runtime/test/pom.xml | 45 +-
.../runtime/common/plan/TestPlanGenerator.java | 4 +-
457 files changed, 5561 insertions(+), 4674 deletions(-)
diff --git a/bin/json2dot.py b/bin/json2dot.py
index e42943d..e82277a 100755
--- a/bin/json2dot.py
+++ b/bin/json2dot.py
@@ -22,306 +22,356 @@ json2dot.py: Generates Graphviz representation of Nemo DAG::toString
This file is used as backend for https://service.jangho.io/nemo-dag/
'''
-import sys
import json
import re
+import sys
nextIdx = 0
+
def propertiesToString(properties):
- return '<BR/>'.join(['{}={}'.format(re.sub('Property$', '', item[0].split('.')[-1]), item[1]) for item in sorted(properties.items())])
+ return '<BR/>'.join(
+ ['{}={}'.format(re.sub('Property$', '', item[0].split('.')[-1]), item[1]) for item in sorted(properties.items())])
+
def getIdx():
- global nextIdx
- nextIdx += 1
- return nextIdx
+ global nextIdx
+ nextIdx += 1
+ return nextIdx
+
def stateToColor(state):
+ try:
+ return {'READY': '#fffbe2',
+ 'EXECUTING': '#e2fbff',
+ 'COMPLETE': '#e2ffe5',
+ 'FAILED_RECOVERABLE': '#ffe2e2',
+ 'FAILED_UNRECOVERABLE': '#e2e2e2'}[state]
+ except:
+ return 'white'
+
+
+class PlanState:
+ def __init__(self, data):
+ self.id = data['planId']
+ self.stages = {}
+ for stage in data['stages']:
+ self.stages[stage['id']] = StageState(stage)
+
+ @classmethod
+ def empty(cls):
+ return cls({'planId': None, 'stages': []})
+
+ def get(self, id):
try:
- return {'READY': '#fffbe2',
- 'EXECUTING': '#e2fbff',
- 'COMPLETE': '#e2ffe5',
- 'FAILED_RECOVERABLE': '#ffe2e2',
- 'FAILED_UNRECOVERABLE': '#e2e2e2'}[state]
+ return self.stages[id]
except:
- return 'white'
+ return StageState.empty()
-class PlanState:
- def __init__(self, data):
- self.id = data['planId']
- self.stages = {}
- for stage in data['stages']:
- self.stages[stage['id']] = StageState(stage)
- @classmethod
- def empty(cls):
- return cls({'planId': None, 'stages': []})
- def get(self, id):
- try:
- return self.stages[id]
- except:
- return StageState.empty()
class StageState:
- def __init__(self, data):
- self.id = data['id']
- self.state = data['state']
- self.tasks = {}
- for irVertex in data['tasks']:
- self.tasks[irVertex['id']] = TaskState(irVertex)
- @classmethod
- def empty(cls):
- return cls({'id': None, 'state': None, 'tasks': []})
- def get(self, id):
- try:
- return self.tasks[id]
- except:
- return TaskState.empty()
- @property
- def taskStateSummary(self):
- stateToNumTasks = dict()
- for taskState in self.tasks.values():
- before = stateToNumTasks.get(taskState.state, 0)
- stateToNumTasks[taskState.state] = before + 1
- return '\\n'.join(['{}: {}'.format(state, stateToNumTasks[state])
- for state in stateToNumTasks.keys()])
+ def __init__(self, data):
+ self.id = data['id']
+ self.state = data['state']
+ self.tasks = {}
+ for irVertex in data['tasks']:
+ self.tasks[irVertex['id']] = TaskState(irVertex)
+
+ @classmethod
+ def empty(cls):
+ return cls({'id': None, 'state': None, 'tasks': []})
+
+ def get(self, id):
+ try:
+ return self.tasks[id]
+ except:
+ return TaskState.empty()
+
+ @property
+ def taskStateSummary(self):
+ stateToNumTasks = dict()
+ for taskState in self.tasks.values():
+ before = stateToNumTasks.get(taskState.state, 0)
+ stateToNumTasks[taskState.state] = before + 1
+ return '\\n'.join(['{}: {}'.format(state, stateToNumTasks[state])
+ for state in stateToNumTasks.keys()])
+
class TaskState:
- def __init__(self, data):
- self.id = data['id']
- self.state = data['state']
- @classmethod
- def empty(cls):
- return cls({'id': None, 'state': None})
+ def __init__(self, data):
+ self.id = data['id']
+ self.state = data['state']
+
+ @classmethod
+ def empty(cls):
+ return cls({'id': None, 'state': None})
+
class DAG:
- '''
- A class for converting DAG to Graphviz representation.
- JSON representation should be formatted like what toString method in DAG.java does.
- '''
- def __init__(self, dag, planState):
- self.vertices = {}
- self.edges = []
- for vertex in dag['vertices']:
- self.vertices[vertex['id']] = Vertex(vertex['id'], vertex['properties'], planState.get(vertex['id']))
- for edge in dag['edges']:
- self.edges.append(Edge(self.vertices[edge['src']], self.vertices[edge['dst']], edge['properties']))
- @property
- def dot(self):
- dot = ''
- for vertex in self.vertices.values():
- dot += vertex.dot
- for edge in self.edges:
- dot += edge.dot
- return dot
+ '''
+ A class for converting DAG to Graphviz representation.
+ JSON representation should be formatted like what toString method in DAG.java does.
+ '''
+
+ def __init__(self, dag, planState):
+ self.vertices = {}
+ self.edges = []
+ for vertex in dag['vertices']:
+ self.vertices[vertex['id']] = Vertex(vertex['id'], vertex['properties'], planState.get(vertex['id']))
+ for edge in dag['edges']:
+ self.edges.append(Edge(self.vertices[edge['src']], self.vertices[edge['dst']], edge['properties']))
+
+ @property
+ def dot(self):
+ dot = ''
+ for vertex in self.vertices.values():
+ dot += vertex.dot
+ for edge in self.edges:
+ dot += edge.dot
+ return dot
+
def Vertex(id, properties, state):
+ try:
+ return Stage(id, properties, state)
+ except:
+ pass
+ try:
+ return LoopVertex(id, properties)
+ except:
+ pass
+ return NormalVertex(id, properties, state)
+
+
+class NormalVertex:
+ def __init__(self, id, properties, state):
+ self.id = id
+ self.properties = properties
+ self.idx = getIdx()
+ self.state = state.state
+
+ @property
+ def dot(self):
+ color = 'black'
+ try:
+ placement = self.properties['executionProperties'][
+ 'org.apache.nemo.common.ir.vertex.executionproperty.ResourcePriorityProperty']
+ if (placement == 'Transient'):
+ color = 'orange'
+ if (placement == 'Reserved'):
+ color = 'green'
+ except:
+ pass
+ label = self.id
+ if self.state is not None:
+ label += '<BR/>({})'.format(self.state)
+ try:
+ label += '<BR/>{}'.format(self.properties['source'])
+ except:
+ pass
try:
- return Stage(id, properties, state)
+ transform = self.properties['transform'].split(':')
+ transform_name = transform[0]
+ try:
+ class_name = transform[1].split('{')[0].split('.')[-1].split('$')[0].split('@')[0]
+ except IndexError:
+ class_name = '?'
+ label += '<BR/>{}:{}'.format(transform_name, class_name)
except:
- pass
+ pass
+ if ('class' in self.properties and self.properties['class'] == 'AggregationBarrierVertex'):
+ shape = ', shape=box'
+ label += '<BR/>AggregationBarrier'
+ else:
+ shape = ''
try:
- return LoopVertex(id, properties)
+ label += '<BR/><FONT POINT-SIZE=\'10\'>{}</FONT>'.format(
+ propertiesToString(self.properties['executionProperties']))
except:
- pass
- return NormalVertex(id, properties, state)
+ pass
+ dot = '{} [label=<{}>, color={}, style=filled, fillcolor="{}"{}];'.format(self.idx, label, color,
+ stateToColor(self.state), shape)
+ return dot
+
+ @property
+ def oneVertex(self):
+ return self
+
+ @property
+ def logicalEnd(self):
+ return self.idx
-class NormalVertex:
- def __init__(self, id, properties, state):
- self.id = id
- self.properties = properties
- self.idx = getIdx()
- self.state = state.state
- @property
- def dot(self):
- color = 'black'
- try:
- placement = self.properties['executionProperties']['org.apache.nemo.common.ir.vertex.executionproperty.ResourcePriorityProperty']
- if (placement == 'Transient'):
- color = 'orange'
- if (placement == 'Reserved'):
- color = 'green'
- except:
- pass
- label = self.id
- if self.state is not None:
- label += '<BR/>({})'.format(self.state)
- try:
- label += '<BR/>{}'.format(self.properties['source'])
- except:
- pass
- try:
- transform = self.properties['transform'].split(':')
- transform_name = transform[0]
- try:
- class_name = transform[1].split('{')[0].split('.')[-1].split('$')[0].split('@')[0]
- except IndexError:
- class_name = '?'
- label += '<BR/>{}:{}'.format(transform_name, class_name)
- except:
- pass
- if ('class' in self.properties and self.properties['class'] == 'AggregationBarrierVertex'):
- shape = ', shape=box'
- label += '<BR/>AggregationBarrier'
- else:
- shape = ''
- try:
- label += '<BR/><FONT POINT-SIZE=\'10\'>{}</FONT>'.format(propertiesToString(self.properties['executionProperties']))
- except:
- pass
- dot = '{} [label=<{}>, color={}, style=filled, fillcolor="{}"{}];'.format(self.idx, label, color, stateToColor(self.state), shape)
- return dot
- @property
- def oneVertex(self):
- return self
- @property
- def logicalEnd(self):
- return self.idx
class LoopVertex:
- def __init__(self, id, properties):
- self.id = id
- self.dag = DAG(properties['DAG'], PlanState.empty())
- self.remaining_iteration = properties['remainingIteration']
- self.executionProperties = properties['executionProperties']
- self.incoming = properties['dagIncomingEdges']
- self.outgoing = properties['dagOutgoingEdges']
- self.edgeMapping = properties['edgeWithLoopToEdgeWithInternalVertex']
- self.idx = getIdx()
- @property
- def dot(self):
- label = self.id
- try:
- label += '<BR/><FONT POINT-SIZE=\'10\'>{}</FONT>'.format(propertiesToString(self.executionProperties))
- except:
- pass
- label += '<BR/>(Remaining iteration: {})'.format(self.remaining_iteration)
- dot = 'subgraph cluster_{} {{'.format(self.idx)
- dot += 'label = "{}";'.format(label)
- dot += self.dag.dot
- dot += '}'
- return dot
- @property
- def oneVertex(self):
- return next(iter(self.dag.vertices.values())).oneVertex
- @property
- def logicalEnd(self):
- return 'cluster_{}'.format(self.idx)
- def internalSrcFor(self, edgeWithLoopId):
- edgeId = self.edgeMapping[edgeWithLoopId]
- vertexId = list(filter(lambda v: edgeId in self.outgoing[v], self.outgoing))[0]
- return self.dag.vertices[vertexId]
- def internalDstFor(self, edgeWithLoopId):
- edgeId = self.edgeMapping[edgeWithLoopId]
- vertexId = list(filter(lambda v: edgeId in self.incoming[v], self.incoming))[0]
- return self.dag.vertices[vertexId]
+ def __init__(self, id, properties):
+ self.id = id
+ self.dag = DAG(properties['DAG'], PlanState.empty())
+ self.remaining_iteration = properties['remainingIteration']
+ self.executionProperties = properties['executionProperties']
+ self.incoming = properties['dagIncomingEdges']
+ self.outgoing = properties['dagOutgoingEdges']
+ self.edgeMapping = properties['edgeWithLoopToEdgeWithInternalVertex']
+ self.idx = getIdx()
+
+ @property
+ def dot(self):
+ label = self.id
+ try:
+ label += '<BR/><FONT POINT-SIZE=\'10\'>{}</FONT>'.format(propertiesToString(self.executionProperties))
+ except:
+ pass
+ label += '<BR/>(Remaining iteration: {})'.format(self.remaining_iteration)
+ dot = 'subgraph cluster_{} {{'.format(self.idx)
+ dot += 'label = "{}";'.format(label)
+ dot += self.dag.dot
+ dot += '}'
+ return dot
+
+ @property
+ def oneVertex(self):
+ return next(iter(self.dag.vertices.values())).oneVertex
+
+ @property
+ def logicalEnd(self):
+ return 'cluster_{}'.format(self.idx)
+
+ def internalSrcFor(self, edgeWithLoopId):
+ edgeId = self.edgeMapping[edgeWithLoopId]
+ vertexId = list(filter(lambda v: edgeId in self.outgoing[v], self.outgoing))[0]
+ return self.dag.vertices[vertexId]
+
+ def internalDstFor(self, edgeWithLoopId):
+ edgeId = self.edgeMapping[edgeWithLoopId]
+ vertexId = list(filter(lambda v: edgeId in self.incoming[v], self.incoming))[0]
+ return self.dag.vertices[vertexId]
+
class Stage:
- def __init__(self, id, properties, state):
- self.id = id
- self.properties = properties
- self.stageDAG = DAG(properties['irDag'], PlanState.empty())
- self.idx = getIdx()
- self.state = state
- self.executionProperties = self.properties['executionProperties']
- @property
- def dot(self):
- if self.state.state is None:
- state = ''
- else:
- state = ' ({})'.format(self.state.state)
- label = '{}{}'.format(self.id, state)
- if self.state.tasks:
- label += '<BR/><BR/>{} Task(s):<BR/>{}'.format(len(self.state.tasks), self.state.taskStateSummary)
- label += '<BR/><FONT POINT-SIZE=\'10\'>{}</FONT>'.format(propertiesToString(self.executionProperties))
- dot = 'subgraph cluster_{} {{'.format(self.idx)
- dot += 'label = <{}>;'.format(label)
- dot += 'color=red; bgcolor="{}";'.format(stateToColor(self.state.state))
- dot += self.stageDAG.dot
- dot += '}'
- return dot
- @property
- def oneVertex(self):
- return next(iter(self.stageDAG.vertices.values())).oneVertex
- @property
- def logicalEnd(self):
- return 'cluster_{}'.format(self.idx)
+ def __init__(self, id, properties, state):
+ self.id = id
+ self.properties = properties
+ self.stageDAG = DAG(properties['irDag'], PlanState.empty())
+ self.idx = getIdx()
+ self.state = state
+ self.executionProperties = self.properties['executionProperties']
+
+ @property
+ def dot(self):
+ if self.state.state is None:
+ state = ''
+ else:
+ state = ' ({})'.format(self.state.state)
+ label = '{}{}'.format(self.id, state)
+ if self.state.tasks:
+ label += '<BR/><BR/>{} Task(s):<BR/>{}'.format(len(self.state.tasks), self.state.taskStateSummary)
+ label += '<BR/><FONT POINT-SIZE=\'10\'>{}</FONT>'.format(propertiesToString(self.executionProperties))
+ dot = 'subgraph cluster_{} {{'.format(self.idx)
+ dot += 'label = <{}>;'.format(label)
+ dot += 'color=red; bgcolor="{}";'.format(stateToColor(self.state.state))
+ dot += self.stageDAG.dot
+ dot += '}'
+ return dot
+
+ @property
+ def oneVertex(self):
+ return next(iter(self.stageDAG.vertices.values())).oneVertex
+
+ @property
+ def logicalEnd(self):
+ return 'cluster_{}'.format(self.idx)
+
def Edge(src, dst, properties):
+ try:
+ return StageEdge(src, dst, properties)
+ except:
+ pass
+ try:
+ return RuntimeEdge(src, dst, properties)
+ except:
+ pass
+ try:
+ return IREdge(src, dst, properties)
+ except:
+ pass
+ return NormalEdge(src, dst, properties)
+
+
+class NormalEdge:
+ def __init__(self, src, dst, properties):
+ self.src = src
+ self.dst = dst
+
+ @property
+ def dot(self):
+ return '{} -> {} [ltail = {}, lhead = {}];'.format(self.src.oneVertex.idx, self.dst.oneVertex.idx,
+ self.src.logicalEnd, self.dst.logicalEnd)
+
+
+class IREdge:
+ def __init__(self, src, dst, properties):
+ self.src = src
+ self.dst = dst
+ self.id = properties['id']
+ self.executionProperties = properties['executionProperties']
+
+ @property
+ def dot(self):
+ src = self.src
+ dst = self.dst
try:
- return StageEdge(src, dst, properties)
- except:
- pass
- try:
- return RuntimeEdge(src, dst, properties)
+ src = src.internalSrcFor(self.id)
except:
- pass
+ pass
try:
- return IREdge(src, dst, properties)
+ dst = dst.internalDstFor(self.id)
except:
- pass
- return NormalEdge(src, dst, properties)
-
-class NormalEdge:
- def __init__(self, src, dst, properties):
- self.src = src
- self.dst = dst
- @property
- def dot(self):
- return '{} -> {} [ltail = {}, lhead = {}];'.format(self.src.oneVertex.idx, self.dst.oneVertex.idx,
- self.src.logicalEnd, self.dst.logicalEnd)
+ pass
+ label = '{}<BR/><FONT POINT-SIZE=\'8\'>{}</FONT>'.format(self.id, propertiesToString(self.executionProperties))
+ return '{} -> {} [ltail = {}, lhead = {}, label = <{}>];'.format(src.oneVertex.idx,
+ dst.oneVertex.idx, src.logicalEnd, dst.logicalEnd,
+ label)
-class IREdge:
- def __init__(self, src, dst, properties):
- self.src = src
- self.dst = dst
- self.id = properties['id']
- self.executionProperties = properties['executionProperties']
- @property
- def dot(self):
- src = self.src
- dst = self.dst
- try:
- src = src.internalSrcFor(self.id)
- except:
- pass
- try:
- dst = dst.internalDstFor(self.id)
- except:
- pass
- label = '{}<BR/><FONT POINT-SIZE=\'8\'>{}</FONT>'.format(self.id, propertiesToString(self.executionProperties))
- return '{} -> {} [ltail = {}, lhead = {}, label = <{}>];'.format(src.oneVertex.idx,
- dst.oneVertex.idx, src.logicalEnd, dst.logicalEnd, label)
class StageEdge:
- def __init__(self, src, dst, properties):
- self.src = src.stageDAG.vertices[properties['externalSrcVertexId']]
- self.dst = dst.stageDAG.vertices[properties['externalDstVertexId']]
- self.runtimeEdgeId = properties['runtimeEdgeId']
- self.executionProperties = properties['executionProperties']
- @property
- def dot(self):
- label = '{}<BR/><FONT POINT-SIZE=\'8\'>{}</FONT>'.format(self.runtimeEdgeId, propertiesToString(self.executionProperties))
- return '{} -> {} [ltail = {}, lhead = {}, label = <{}>];'.format(self.src.oneVertex.idx,
- self.dst.oneVertex.idx, self.src.logicalEnd, self.dst.logicalEnd, label)
+ def __init__(self, src, dst, properties):
+ self.src = src.stageDAG.vertices[properties['externalSrcVertexId']]
+ self.dst = dst.stageDAG.vertices[properties['externalDstVertexId']]
+ self.runtimeEdgeId = properties['runtimeEdgeId']
+ self.executionProperties = properties['executionProperties']
+
+ @property
+ def dot(self):
+ label = '{}<BR/><FONT POINT-SIZE=\'8\'>{}</FONT>'.format(self.runtimeEdgeId,
+ propertiesToString(self.executionProperties))
+ return '{} -> {} [ltail = {}, lhead = {}, label = <{}>];'.format(self.src.oneVertex.idx,
+ self.dst.oneVertex.idx, self.src.logicalEnd,
+ self.dst.logicalEnd, label)
+
class RuntimeEdge:
- def __init__(self, src, dst, properties):
- self.src = src
- self.dst = dst
- self.runtimeEdgeId = properties['runtimeEdgeId']
- self.executionProperties = properties['executionProperties']
- @property
- def dot(self):
- label = '{}<BR/><FONT POINT-SIZE=\'8\'>{}</FONT>'.format(self.runtimeEdgeId, propertiesToString(self.executionProperties))
- return '{} -> {} [ltail = {}, lhead = {}, label = <{}>];'.format(self.src.oneVertex.idx,
- self.dst.oneVertex.idx, self.src.logicalEnd, self.dst.logicalEnd, label)
+ def __init__(self, src, dst, properties):
+ self.src = src
+ self.dst = dst
+ self.runtimeEdgeId = properties['runtimeEdgeId']
+ self.executionProperties = properties['executionProperties']
+
+ @property
+ def dot(self):
+ label = '{}<BR/><FONT POINT-SIZE=\'8\'>{}</FONT>'.format(self.runtimeEdgeId,
+ propertiesToString(self.executionProperties))
+ return '{} -> {} [ltail = {}, lhead = {}, label = <{}>];'.format(self.src.oneVertex.idx,
+ self.dst.oneVertex.idx, self.src.logicalEnd,
+ self.dst.logicalEnd, label)
+
def jsonToDot(jsonDict):
- try:
- dag = DAG(jsonDict['dag'], PlanState(jsonDict['planState']))
- except:
- dag = DAG(jsonDict, PlanState.empty())
- return 'digraph dag {compound=true; nodesep=1.0; forcelabels=true;' + dag.dot + '}'
+ try:
+ dag = DAG(jsonDict['dag'], PlanState(jsonDict['planState']))
+ except:
+ dag = DAG(jsonDict, PlanState.empty())
+ return 'digraph dag {compound=true; nodesep=1.0; forcelabels=true;' + dag.dot + '}'
+
if __name__ == "__main__":
- print(jsonToDot(json.loads(sys.stdin.read())))
+ print(jsonToDot(json.loads(sys.stdin.read())))
diff --git a/bin/metric-parser.py b/bin/metric-parser.py
index 898c34c..f7ef572 100755
--- a/bin/metric-parser.py
+++ b/bin/metric-parser.py
@@ -17,71 +17,78 @@
# specific language governing permissions and limitations
# under the License.
-import sys
-import os
import json
import numpy as np
+import os
+import sys
+
def main():
- try:
- filepath = sys.argv[1]
- except IndexError:
- print("Please provide the file path for the metric log file.")
- else:
- if not os.path.isfile(filepath):
- print("File path {} does not exist. Exiting...".format(filepath))
- sys.exit()
+ try:
+ filepath = sys.argv[1]
+ except IndexError:
+ print("Please provide the file path for the metric log file.")
+ else:
+ if not os.path.isfile(filepath):
+ print("File path {} does not exist. Exiting...".format(filepath))
+ sys.exit()
- metricDictionary = dict()
- vertexToMetricDict = dict()
- with open(filepath, 'r') as fp:
- for line in fp:
- metricInJson = json.loads(line)
- metricKey = metricInJson["computationUnitId"]
- metricDictionary[metricKey] = metricInJson["metricList"]
- if metricKey.find('Task-vertex-') != -1: # Vertex metric
- vertexIdSuffix = metricKey.split('Task-vertex-')[1]
- if vertexIdSuffix.find('_') != -1: # physical level metric
- vertexId = 'vertex-' + vertexIdSuffix.split('_')[0]
- metricDictList = metricDictionary[metricKey]
- if isinstance(metricDictList, dict):
- metricDictList = [metricDictList]
- for metricDict in metricDictList:
- for key, value in metricDict.items():
- if (key != 'EndTime') & (key != 'StartTime'):
- vertexMetricDict = vertexToMetricDict.get(vertexId, dict())
- vertexMetricDictValueList = vertexMetricDict.get(key, [])
- vertexMetricDictValueList.append(value)
- vertexMetricDict[key] = vertexMetricDictValueList
- vertexToMetricDict[vertexId] = vertexMetricDict
+ metricDictionary = dict()
+ vertexToMetricDict = dict()
+ with open(filepath, 'r') as fp:
+ for line in fp:
+ metricInJson = json.loads(line)
+ metricKey = metricInJson["computationUnitId"]
+ metricDictionary[metricKey] = metricInJson["metricList"]
+ if metricKey.find('Task-vertex-') != -1: # Vertex metric
+ vertexIdSuffix = metricKey.split('Task-vertex-')[1]
+ if vertexIdSuffix.find('_') != -1: # physical level metric
+ vertexId = 'vertex-' + vertexIdSuffix.split('_')[0]
+ metricDictList = metricDictionary[metricKey]
+ if isinstance(metricDictList, dict):
+ metricDictList = [metricDictList]
+ for metricDict in metricDictList:
+ for key, value in metricDict.items():
+ if (key != 'EndTime') & (key != 'StartTime'):
+ vertexMetricDict = vertexToMetricDict.get(vertexId, dict())
+ vertexMetricDictValueList = vertexMetricDict.get(key, [])
+ vertexMetricDictValueList.append(value)
+ vertexMetricDict[key] = vertexMetricDictValueList
+ vertexToMetricDict[vertexId] = vertexMetricDict
- query_metric = True
- while(query_metric):
- user_input = input("1 - View metric for a computation unit, 2 - View metric for all IR vertices, 3 - exit: ")
- if user_input == "1":
- computationUnitId = input("Enter computation unit ID: ")
- for metric in metricDictionary[computationUnitId]:
- print(metric)
- elif user_input == "2":
- for vertexId, metricDict in sorted(vertexToMetricDict.items()):
- print(vertexId)
- metricKeys, valuesMin, valuesMedian, valuesMax, valuesMean, valuesSum = ['Metric'], ['Min'], ['Median'], ['Max'], ['Mean'], ['Total']
- for metricKey, metricValues in metricDict.items():
- metricKeys.append(metricKey)
- valuesMin.append(str(np.min(metricValues)))
- valuesMedian.append(str(np.median(metricValues)))
- valuesMax.append(str(np.max(metricValues)))
- valuesMean.append(str(np.mean(metricValues)))
- valuesSum.append(str(np.sum(metricValues)))
- padding = 1
- widthKey, widthMin, widthMedian, widthMax, widthMean, widthSum = map(lambda x:len(max(x, key=len)) + padding, [metricKeys, valuesMin, valuesMedian, valuesMax, valuesMean, valuesSum])
- templete = '{:<%s} {:<%s} {:<%s} {:<%s} {:<%s} {:<%s}' % (widthKey, widthMin, widthMedian, widthMax, widthMean, widthSum)
- for metricKey, valueMin, valueMedian, valueMax, valueMean, valueSum in zip(metricKeys, valuesMin, valuesMedian, valuesMax, valuesMean, valuesSum):
- print(templete.format(metricKey, valueMin, valueMedian, valueMax, valueMean, valueSum))
- else:
- print ("Exiting metric parser")
- query_metric = False
+ query_metric = True
+ while (query_metric):
+ user_input = input("1 - View metric for a computation unit, 2 - View metric for all IR vertices, 3 - exit: ")
+ if user_input == "1":
+ computationUnitId = input("Enter computation unit ID: ")
+ for metric in metricDictionary[computationUnitId]:
+ print(metric)
+ elif user_input == "2":
+ for vertexId, metricDict in sorted(vertexToMetricDict.items()):
+ print(vertexId)
+ metricKeys, valuesMin, valuesMedian, valuesMax, valuesMean, valuesSum = ['Metric'], ['Min'], ['Median'], [
+ 'Max'], ['Mean'], ['Total']
+ for metricKey, metricValues in metricDict.items():
+ metricKeys.append(metricKey)
+ valuesMin.append(str(np.min(metricValues)))
+ valuesMedian.append(str(np.median(metricValues)))
+ valuesMax.append(str(np.max(metricValues)))
+ valuesMean.append(str(np.mean(metricValues)))
+ valuesSum.append(str(np.sum(metricValues)))
+ padding = 1
+ widthKey, widthMin, widthMedian, widthMax, widthMean, widthSum = map(lambda x: len(max(x, key=len)) + padding,
+ [metricKeys, valuesMin, valuesMedian,
+ valuesMax, valuesMean, valuesSum])
+ templete = '{:<%s} {:<%s} {:<%s} {:<%s} {:<%s} {:<%s}' % (
+ widthKey, widthMin, widthMedian, widthMax, widthMean, widthSum)
+ for metricKey, valueMin, valueMedian, valueMax, valueMean, valueSum in zip(metricKeys, valuesMin,
+ valuesMedian, valuesMax,
+ valuesMean, valuesSum):
+ print(templete.format(metricKey, valueMin, valueMedian, valueMax, valueMean, valueSum))
+ else:
+ print("Exiting metric parser")
+ query_metric = False
if __name__ == '__main__':
- main()
+ main()
diff --git a/checkstyle.xml b/checkstyle.xml
index c7f8761..7f97abc 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -18,8 +18,8 @@ specific language governing permissions and limitations
under the License.
-->
<!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
- "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<!--
Checkstyle configuration that checks the sun coding conventions from:
@@ -38,148 +38,148 @@ under the License.
-->
<module name="Checker">
- <!--
- If you set the basedir property below, then all reported file
- names will be relative to the specified directory. See
- http://checkstyle.sourceforge.net/5.x/config.html#Checker
- <property name="basedir" value="${basedir}"/>
- -->
- <property name="charset" value="UTF-8"/>
-
- <property name="fileExtensions" value="java, properties, xml"/>
-
- <!-- Checks that a package-info.java file exists for each package. -->
- <!-- See http://checkstyle.sf.net/config_javadoc.html#JavadocPackage -->
- <!--<module name="JavadocPackage"/>-->
-
- <!-- Checks whether files end with a new line. -->
- <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
- <module name="NewlineAtEndOfFile"/>
-
- <!-- Checks that property files contain the same keys. -->
- <!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
- <module name="Translation"/>
+ <!--
+ If you set the basedir property below, then all reported file
+ names will be relative to the specified directory. See
+ http://checkstyle.sourceforge.net/5.x/config.html#Checker
+ <property name="basedir" value="${basedir}"/>
+ -->
+ <property name="charset" value="UTF-8"/>
+
+ <property name="fileExtensions" value="java, properties, xml"/>
+
+ <!-- Checks that a package-info.java file exists for each package. -->
+ <!-- See http://checkstyle.sf.net/config_javadoc.html#JavadocPackage -->
+ <!--<module name="JavadocPackage"/>-->
+
+ <!-- Checks whether files end with a new line. -->
+ <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
+ <module name="NewlineAtEndOfFile"/>
+
+ <!-- Checks that property files contain the same keys. -->
+ <!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
+ <module name="Translation"/>
+
+ <!-- Checks for Size Violations. -->
+ <!-- See http://checkstyle.sf.net/config_sizes.html -->
+ <module name="FileLength"/>
+
+ <!-- Checks for whitespace -->
+ <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+ <module name="FileTabCharacter"/>
+
+ <!-- Miscellaneous other checks. -->
+ <!-- See http://checkstyle.sf.net/config_misc.html -->
+ <module name="RegexpSingleline">
+ <property name="format" value="\s+$"/>
+ <property name="minimum" value="0"/>
+ <property name="maximum" value="0"/>
+ <property name="message" value="Line has trailing spaces."/>
+ </module>
+
+ <module name="SuppressWarningsFilter"/>
+
+ <module name="TreeWalker">
+
+ <!-- Checks for Javadoc comments. -->
+ <!-- See http://checkstyle.sf.net/config_javadoc.html -->
+ <module name="JavadocMethod">
+ <property name="severity" value="warning"/>
+ </module>
+ <module name="JavadocType"/>
+ <!--<module name="JavadocVariable"/>-->
+ <module name="JavadocStyle"/>
+
+ <!-- Checks for Naming Conventions. -->
+ <!-- See http://checkstyle.sf.net/config_naming.html -->
+ <module name="ConstantName"/>
+ <module name="LocalFinalVariableName"/>
+ <module name="LocalVariableName"/>
+ <module name="MemberName"/>
+ <module name="MethodName"/>
+ <module name="PackageName"/>
+ <module name="ParameterName"/>
+ <module name="StaticVariableName"/>
+ <module name="TypeName"/>
+
+ <!-- Checks for imports -->
+ <!-- See http://checkstyle.sf.net/config_import.html -->
+ <!--<module name="AvoidStarImport"/>-->
+ <module name="IllegalImport"/> <!-- defaults to sun.* packages -->
+ <module name="RedundantImport"/>
+ <module name="UnusedImports">
+ <property name="processJavadoc" value="false"/>
+ </module>
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
- <module name="FileLength"/>
+ <module name="LineLength">
+ <property name="max" value="120"/>
+ <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
+ </module>
+ <module name="MethodLength"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
- <module name="FileTabCharacter"/>
+ <module name="EmptyForIteratorPad"/>
+ <module name="GenericWhitespace"/>
+ <module name="MethodParamPad"/>
+ <module name="NoWhitespaceAfter"/>
+ <module name="NoWhitespaceBefore"/>
+ <module name="OperatorWrap"/>
+ <module name="ParenPad"/>
+ <module name="TypecastParenPad"/>
+ <module name="WhitespaceAfter"/>
+ <module name="WhitespaceAround"/>
+
+ <!-- Modifier Checks -->
+ <!-- See http://checkstyle.sf.net/config_modifiers.html -->
+ <module name="ModifierOrder"/>
+ <module name="RedundantModifier"/>
+
+ <!-- Checks for blocks. You know, those {}'s -->
+ <!-- See http://checkstyle.sf.net/config_blocks.html -->
+ <module name="AvoidNestedBlocks"/>
+ <module name="EmptyBlock"/>
+ <module name="LeftCurly"/>
+ <module name="NeedBraces"/>
+ <module name="RightCurly"/>
+
+ <!-- Checks for common coding problems -->
+ <!-- See http://checkstyle.sf.net/config_coding.html -->
+ <!--<module name="AvoidInlineConditionals"/>-->
+ <module name="EmptyStatement"/>
+ <module name="EqualsHashCode"/>
+ <module name="HiddenField">
+ <property name="ignoreSetter" value="true"/>
+ <property name="ignoreConstructorParameter" value="true"/>
+ </module>
+ <module name="IllegalInstantiation"/>
+ <module name="InnerAssignment"/>
+ <!--<module name="MagicNumber"/>-->
+ <module name="MissingSwitchDefault"/>
+ <module name="SimplifyBooleanExpression"/>
+ <module name="SimplifyBooleanReturn"/>
+
+ <!-- Checks for class design -->
+ <!-- See http://checkstyle.sf.net/config_design.html -->
+ <module name="DesignForExtension"/>
+ <module name="FinalClass"/>
+ <module name="HideUtilityClassConstructor"/>
+ <module name="InterfaceIsType"/>
+ <module name="VisibilityModifier"/>
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
- <module name="RegexpSingleline">
- <property name="format" value="\s+$"/>
- <property name="minimum" value="0"/>
- <property name="maximum" value="0"/>
- <property name="message" value="Line has trailing spaces."/>
- </module>
+ <module name="ArrayTypeStyle"/>
+ <module name="FinalParameters"/>
+ <module name="TodoComment"/>
+ <module name="UpperEll"/>
- <module name="SuppressWarningsFilter" />
-
- <module name="TreeWalker">
-
- <!-- Checks for Javadoc comments. -->
- <!-- See http://checkstyle.sf.net/config_javadoc.html -->
- <module name="JavadocMethod">
- <property name="severity" value="warning"/>
- </module>
- <module name="JavadocType"/>
- <!--<module name="JavadocVariable"/>-->
- <module name="JavadocStyle"/>
-
- <!-- Checks for Naming Conventions. -->
- <!-- See http://checkstyle.sf.net/config_naming.html -->
- <module name="ConstantName"/>
- <module name="LocalFinalVariableName"/>
- <module name="LocalVariableName"/>
- <module name="MemberName"/>
- <module name="MethodName"/>
- <module name="PackageName"/>
- <module name="ParameterName"/>
- <module name="StaticVariableName"/>
- <module name="TypeName"/>
-
- <!-- Checks for imports -->
- <!-- See http://checkstyle.sf.net/config_import.html -->
- <!--<module name="AvoidStarImport"/>-->
- <module name="IllegalImport"/> <!-- defaults to sun.* packages -->
- <module name="RedundantImport"/>
- <module name="UnusedImports">
- <property name="processJavadoc" value="false"/>
- </module>
-
- <!-- Checks for Size Violations. -->
- <!-- See http://checkstyle.sf.net/config_sizes.html -->
- <module name="LineLength">
- <property name="max" value="120"/>
- <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
- </module>
- <module name="MethodLength"/>
-
- <!-- Checks for whitespace -->
- <!-- See http://checkstyle.sf.net/config_whitespace.html -->
- <module name="EmptyForIteratorPad"/>
- <module name="GenericWhitespace"/>
- <module name="MethodParamPad"/>
- <module name="NoWhitespaceAfter"/>
- <module name="NoWhitespaceBefore"/>
- <module name="OperatorWrap"/>
- <module name="ParenPad"/>
- <module name="TypecastParenPad"/>
- <module name="WhitespaceAfter"/>
- <module name="WhitespaceAround"/>
-
- <!-- Modifier Checks -->
- <!-- See http://checkstyle.sf.net/config_modifiers.html -->
- <module name="ModifierOrder"/>
- <module name="RedundantModifier"/>
-
- <!-- Checks for blocks. You know, those {}'s -->
- <!-- See http://checkstyle.sf.net/config_blocks.html -->
- <module name="AvoidNestedBlocks"/>
- <module name="EmptyBlock"/>
- <module name="LeftCurly"/>
- <module name="NeedBraces"/>
- <module name="RightCurly"/>
-
- <!-- Checks for common coding problems -->
- <!-- See http://checkstyle.sf.net/config_coding.html -->
- <!--<module name="AvoidInlineConditionals"/>-->
- <module name="EmptyStatement"/>
- <module name="EqualsHashCode"/>
- <module name="HiddenField">
- <property name="ignoreSetter" value="true" />
- <property name="ignoreConstructorParameter" value="true" />
- </module>
- <module name="IllegalInstantiation"/>
- <module name="InnerAssignment"/>
- <!--<module name="MagicNumber"/>-->
- <module name="MissingSwitchDefault"/>
- <module name="SimplifyBooleanExpression"/>
- <module name="SimplifyBooleanReturn"/>
-
- <!-- Checks for class design -->
- <!-- See http://checkstyle.sf.net/config_design.html -->
- <module name="DesignForExtension"/>
- <module name="FinalClass"/>
- <module name="HideUtilityClassConstructor"/>
- <module name="InterfaceIsType"/>
- <module name="VisibilityModifier"/>
-
- <!-- Miscellaneous other checks. -->
- <!-- See http://checkstyle.sf.net/config_misc.html -->
- <module name="ArrayTypeStyle"/>
- <module name="FinalParameters"/>
- <module name="TodoComment"/>
- <module name="UpperEll"/>
-
- <module name="SuppressWarningsHolder" />
- </module>
+ <module name="SuppressWarningsHolder"/>
+ </module>
- <module name="RegexpHeader">
- <property name="headerFile" value="${checkstyle.header.file}" />
- </module>
+ <module name="RegexpHeader">
+ <property name="headerFile" value="${checkstyle.header.file}"/>
+ </module>
</module>
diff --git a/client/pom.xml b/client/pom.xml
index c9439d2..367171b 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -17,60 +17,61 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-project</artifactId>
- <version>0.2-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-project</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
- <artifactId>nemo-client</artifactId>
- <name>Nemo Client</name>
+ <artifactId>nemo-client</artifactId>
+ <name>Nemo Client</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-driver</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-conf</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.reef</groupId>
- <artifactId>reef-runtime-local</artifactId>
- <version>${reef.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.reef</groupId>
- <artifactId>reef-runtime-yarn</artifactId>
- <version>${reef.version}</version>
- </dependency>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-driver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-conf</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.reef</groupId>
+ <artifactId>reef-runtime-local</artifactId>
+ <version>${reef.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.reef</groupId>
+ <artifactId>reef-runtime-yarn</artifactId>
+ <version>${reef.version}</version>
+ </dependency>
- <!-- for nemo-beam-runner -->
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler-frontend-beam</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.auto.service</groupId>
- <artifactId>auto-service</artifactId>
- <version>${auto-service.version}</version>
- <optional>true</optional>
- </dependency>
- </dependencies>
+ <!-- for nemo-beam-runner -->
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler-frontend-beam</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <version>${auto-service.version}</version>
+ <optional>true</optional>
+ </dependency>
+ </dependencies>
<build>
@@ -100,7 +101,7 @@ under the License.
<transformers>
<!-- Required for using beam-hadoop: See https://stackoverflow.com/questions/44365545
-->
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
</transformers>
</configuration>
</execution>
diff --git a/client/src/main/java/org/apache/nemo/client/ClientEndpoint.java b/client/src/main/java/org/apache/nemo/client/ClientEndpoint.java
index e29b351..257d23a 100644
--- a/client/src/main/java/org/apache/nemo/client/ClientEndpoint.java
+++ b/client/src/main/java/org/apache/nemo/client/ClientEndpoint.java
@@ -50,6 +50,7 @@ public abstract class ClientEndpoint {
/**
* Constructor.
+ *
* @param stateTranslator translator to translate between the state of plan and corresponding.
*/
public ClientEndpoint(final StateTranslator stateTranslator) {
@@ -146,7 +147,7 @@ public abstract class ClientEndpoint {
* @return the final state of this job.
*/
public final Enum waitUntilJobFinish(final long timeout,
- final TimeUnit unit) {
+ final TimeUnit unit) {
if (driverEndpoint.get() != null) {
return stateTranslator.translateState(driverEndpoint.get().waitUntilFinish(timeout, unit));
} else {
@@ -162,7 +163,7 @@ public abstract class ClientEndpoint {
if (driverIsConnected) {
final long consumedTime = System.nanoTime() - currentNano;
return stateTranslator.translateState(driverEndpoint.get().
- waitUntilFinish(timeout - unit.convert(consumedTime, TimeUnit.NANOSECONDS), unit));
+ waitUntilFinish(timeout - unit.convert(consumedTime, TimeUnit.NANOSECONDS), unit));
} else {
return PlanState.State.READY;
}
diff --git a/client/src/main/java/org/apache/nemo/client/DriverEndpoint.java b/client/src/main/java/org/apache/nemo/client/DriverEndpoint.java
index 2b52a2c..6deb8e6 100644
--- a/client/src/main/java/org/apache/nemo/client/DriverEndpoint.java
+++ b/client/src/main/java/org/apache/nemo/client/DriverEndpoint.java
@@ -41,8 +41,9 @@ public final class DriverEndpoint {
/**
* Construct an endpoint in driver side.
* This method will be called by {@link ClientEndpoint}.
+ *
* @param planStateManager of running plan.
- * @param clientEndpoint of running plan.
+ * @param clientEndpoint of running plan.
*/
public DriverEndpoint(final PlanStateManager planStateManager,
final ClientEndpoint clientEndpoint) {
@@ -54,6 +55,7 @@ public final class DriverEndpoint {
/**
* Get the current state of the running plan.
* This method will be called by {@link ClientEndpoint}.
+ *
* @return the current state of the running plan.
*/
PlanState.State getState() {
@@ -64,8 +66,9 @@ public final class DriverEndpoint {
* Wait for this plan to be finished and return the final state.
* It wait for at most the given time.
* This method will be called by {@link ClientEndpoint}.
+ *
* @param timeout of waiting.
- * @param unit of the timeout.
+ * @param unit of the timeout.
* @return the final state of this plan.
*/
PlanState.State waitUntilFinish(final long timeout,
@@ -76,6 +79,7 @@ public final class DriverEndpoint {
/**
* Wait for this plan to be finished and return the final state.
* This method will be called by {@link ClientEndpoint}.
+ *
* @return the final state of this plan.
*/
PlanState.State waitUntilFinish() {
diff --git a/client/src/main/java/org/apache/nemo/client/DriverRPCServer.java b/client/src/main/java/org/apache/nemo/client/DriverRPCServer.java
index 29db74c..5d7b998 100644
--- a/client/src/main/java/org/apache/nemo/client/DriverRPCServer.java
+++ b/client/src/main/java/org/apache/nemo/client/DriverRPCServer.java
@@ -48,7 +48,7 @@ import java.util.Map;
@NotThreadSafe
public final class DriverRPCServer {
private final Map<ControlMessage.DriverToClientMessageType, EventHandler<ControlMessage.DriverToClientMessage>>
- handlers = new HashMap<>();
+ handlers = new HashMap<>();
private boolean isRunning = false;
private boolean isShutdown = false;
private Transport transport;
@@ -59,7 +59,8 @@ public final class DriverRPCServer {
/**
* Registers handler for the given type of message.
- * @param type the type of message
+ *
+ * @param type the type of message
* @param handler handler implementation
* @return {@code this}
*/
@@ -88,7 +89,7 @@ public final class DriverRPCServer {
injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, host);
injector.bindVolatileParameter(RemoteConfiguration.Port.class, 0);
injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class,
- new SyncStage<>(new ServerEventHandler()));
+ new SyncStage<>(new ServerEventHandler()));
transport = injector.getInstance(NettyMessagingTransport.class);
LOG.info("DriverRPCServer running at {}", transport.getListeningPort());
isRunning = true;
@@ -120,13 +121,14 @@ public final class DriverRPCServer {
*/
public Configuration getListeningConfiguration() {
return Tang.Factory.getTang().newConfigurationBuilder()
- .bindNamedParameter(JobConf.ClientSideRPCServerHost.class, getListeningHost())
- .bindNamedParameter(JobConf.ClientSideRPCServerPort.class, String.valueOf(getListeningPort()))
- .build();
+ .bindNamedParameter(JobConf.ClientSideRPCServerHost.class, getListeningHost())
+ .bindNamedParameter(JobConf.ClientSideRPCServerPort.class, String.valueOf(getListeningPort()))
+ .build();
}
/**
* Sends a message to driver.
+ *
* @param message message to send
*/
public void send(final ControlMessage.ClientToDriverMessage message) {
@@ -186,6 +188,7 @@ public final class DriverRPCServer {
/**
* Throws a {@link RuntimeException} if the server is shut down, or it has different state than the expected state.
+ *
* @param running the expected state of the server
*/
private void ensureServerState(final boolean running) {
diff --git a/client/src/main/java/org/apache/nemo/client/JobLauncher.java b/client/src/main/java/org/apache/nemo/client/JobLauncher.java
index ba6194b..233ce73 100644
--- a/client/src/main/java/org/apache/nemo/client/JobLauncher.java
+++ b/client/src/main/java/org/apache/nemo/client/JobLauncher.java
@@ -20,6 +20,7 @@ package org.apache.nemo.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import org.apache.commons.lang3.SerializationUtils;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.compiler.backend.nemo.NemoPlanRewriter;
import org.apache.nemo.conf.JobConf;
@@ -27,7 +28,6 @@ import org.apache.nemo.driver.NemoDriver;
import org.apache.nemo.runtime.common.comm.ControlMessage;
import org.apache.nemo.runtime.common.message.MessageEnvironment;
import org.apache.nemo.runtime.common.message.MessageParameters;
-import org.apache.commons.lang3.SerializationUtils;
import org.apache.nemo.runtime.common.plan.PlanRewriter;
import org.apache.nemo.runtime.master.scheduler.Scheduler;
import org.apache.reef.client.DriverConfiguration;
@@ -65,12 +65,12 @@ public final class JobLauncher {
static {
System.out.println(
- "\nPowered by\n"
- + " _ __ \n"
- + " / | / /__ ____ ___ ____ \n"
- + " / |/ / _ \\/ __ `__ \\/ __ \\\n"
- + " / /| / __/ / / / / / /_/ /\n"
- + "/_/ |_/\\___/_/ /_/ /_/\\____/ \n"
+ "\nPowered by\n"
+ + " _ __ \n"
+ + " / | / /__ ____ ___ ____ \n"
+ + " / |/ / _ \\/ __ `__ \\/ __ \\\n"
+ + " / /| / __/ / / / / / /_/ /\n"
+ + "/_/ |_/\\___/_/ /_/ /_/\\____/ \n"
);
}
@@ -114,10 +114,11 @@ public final class JobLauncher {
/**
* Set up the driver, etc. before the actual execution.
+ *
* @param args arguments.
- * @throws InjectionException injection exception from REEF.
+ * @throws InjectionException injection exception from REEF.
* @throws ClassNotFoundException class not found exception.
- * @throws IOException IO exception.
+ * @throws IOException IO exception.
*/
public static void setup(final String[] args) throws InjectionException, ClassNotFoundException, IOException {
// Get Job and Driver Confs
@@ -213,7 +214,7 @@ public final class JobLauncher {
}
/**
- * @param dag the application DAG.
+ * @param dag the application DAG.
* @param jobId job ID.
*/
public static void launchDAG(final IRDAG dag, final String jobId) {
@@ -221,9 +222,9 @@ public final class JobLauncher {
}
/**
- * @param dag the application DAG.
+ * @param dag the application DAG.
* @param broadcastVariables broadcast variables (can be empty).
- * @param jobId job ID.
+ * @param jobId job ID.
*/
public static void launchDAG(final IRDAG dag,
final Map<Serializable, Object> broadcastVariables,
@@ -251,12 +252,12 @@ public final class JobLauncher {
serializedDAG = Base64.getEncoder().encodeToString(SerializationUtils.serialize(dag));
jobDoneLatch = new CountDownLatch(1);
driverRPCServer.send(ControlMessage.ClientToDriverMessage.newBuilder()
- .setType(ControlMessage.ClientToDriverMessageType.LaunchDAG)
- .setLaunchDAG(ControlMessage.LaunchDAGMessage.newBuilder()
- .setDag(serializedDAG)
- .setBroadcastVars(ByteString.copyFrom(SerializationUtils.serialize((Serializable) broadcastVariables)))
- .build())
- .build());
+ .setType(ControlMessage.ClientToDriverMessageType.LaunchDAG)
+ .setLaunchDAG(ControlMessage.LaunchDAGMessage.newBuilder()
+ .setDag(serializedDAG)
+ .setBroadcastVars(ByteString.copyFrom(SerializationUtils.serialize((Serializable) broadcastVariables)))
+ .build())
+ .build());
// Wait for the ExecutionDone message from the driver
try {
@@ -310,10 +311,11 @@ public final class JobLauncher {
/**
* Fetch scheduler configuration.
+ *
* @param jobConf job configuration.
* @return the scheduler configuration.
* @throws ClassNotFoundException exception while finding the class.
- * @throws InjectionException exception while injection (REEF Tang).
+ * @throws InjectionException exception while injection (REEF Tang).
*/
private static Configuration getSchedulerConf(final Configuration jobConf)
throws ClassNotFoundException, InjectionException {
@@ -334,10 +336,10 @@ public final class JobLauncher {
*/
private static Configuration getDriverNcsConf() throws InjectionException {
return Configurations.merge(NameServerConfiguration.CONF.build(),
- LocalNameResolverConfiguration.CONF.build(),
- TANG.newConfigurationBuilder()
- .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
- .build());
+ LocalNameResolverConfiguration.CONF.build(),
+ TANG.newConfigurationBuilder()
+ .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
+ .build());
}
/**
@@ -348,8 +350,8 @@ public final class JobLauncher {
*/
private static Configuration getDriverMessageConf() throws InjectionException {
return TANG.newConfigurationBuilder()
- .bindNamedParameter(MessageParameters.SenderId.class, MessageEnvironment.MASTER_COMMUNICATION_ID)
- .build();
+ .bindNamedParameter(MessageParameters.SenderId.class, MessageEnvironment.MASTER_COMMUNICATION_ID)
+ .build();
}
/**
@@ -364,16 +366,16 @@ public final class JobLauncher {
final String jobId = injector.getNamedInstance(JobConf.JobId.class);
final int driverMemory = injector.getNamedInstance(JobConf.DriverMemMb.class);
return DriverConfiguration.CONF
- .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(NemoDriver.class))
- .set(DriverConfiguration.ON_DRIVER_STARTED, NemoDriver.StartHandler.class)
- .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, NemoDriver.AllocatedEvaluatorHandler.class)
- .set(DriverConfiguration.ON_CONTEXT_ACTIVE, NemoDriver.ActiveContextHandler.class)
- .set(DriverConfiguration.ON_EVALUATOR_FAILED, NemoDriver.FailedEvaluatorHandler.class)
- .set(DriverConfiguration.ON_CONTEXT_FAILED, NemoDriver.FailedContextHandler.class)
- .set(DriverConfiguration.ON_DRIVER_STOP, NemoDriver.DriverStopHandler.class)
- .set(DriverConfiguration.DRIVER_IDENTIFIER, jobId)
- .set(DriverConfiguration.DRIVER_MEMORY, driverMemory)
- .build();
+ .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(NemoDriver.class))
+ .set(DriverConfiguration.ON_DRIVER_STARTED, NemoDriver.StartHandler.class)
+ .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, NemoDriver.AllocatedEvaluatorHandler.class)
+ .set(DriverConfiguration.ON_CONTEXT_ACTIVE, NemoDriver.ActiveContextHandler.class)
+ .set(DriverConfiguration.ON_EVALUATOR_FAILED, NemoDriver.FailedEvaluatorHandler.class)
+ .set(DriverConfiguration.ON_CONTEXT_FAILED, NemoDriver.FailedContextHandler.class)
+ .set(DriverConfiguration.ON_DRIVER_STOP, NemoDriver.DriverStopHandler.class)
+ .set(DriverConfiguration.DRIVER_IDENTIFIER, jobId)
+ .set(DriverConfiguration.DRIVER_MEMORY, driverMemory)
+ .build();
}
/**
@@ -426,12 +428,12 @@ public final class JobLauncher {
switch (deployMode) {
case "local":
return LocalRuntimeConfiguration.CONF
- .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS)
- .build();
+ .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS)
+ .build();
case "yarn":
return YarnClientConfiguration.CONF
- .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class))
- .build();
+ .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class))
+ .build();
default:
throw new UnsupportedOperationException(deployMode);
}
@@ -451,15 +453,15 @@ public final class JobLauncher {
final Class<? extends Name<String>> pathParameter,
final Class<? extends Name<String>> contentsParameter,
final String defaultContent)
- throws InjectionException {
+ throws InjectionException {
final Injector injector = TANG.newInjector(jobConf);
try {
final String path = injector.getNamedInstance(pathParameter);
final String contents = path.isEmpty() ? defaultContent
: new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8);
return TANG.newConfigurationBuilder()
- .bindNamedParameter(contentsParameter, contents)
- .build();
+ .bindNamedParameter(contentsParameter, contents)
+ .build();
} catch (final IOException e) {
throw new RuntimeException(e);
}
diff --git a/client/src/main/java/org/apache/nemo/client/beam/NemoPipelineResult.java b/client/src/main/java/org/apache/nemo/client/beam/NemoPipelineResult.java
index 512f8f1..d983521 100644
--- a/client/src/main/java/org/apache/nemo/client/beam/NemoPipelineResult.java
+++ b/client/src/main/java/org/apache/nemo/client/beam/NemoPipelineResult.java
@@ -18,9 +18,9 @@
*/
package org.apache.nemo.client.beam;
-import org.apache.nemo.client.ClientEndpoint;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.nemo.client.ClientEndpoint;
import org.joda.time.Duration;
import java.io.IOException;
diff --git a/client/src/main/java/org/apache/nemo/client/beam/NemoRunner.java b/client/src/main/java/org/apache/nemo/client/beam/NemoRunner.java
index ccafc3e..81aa18d 100644
--- a/client/src/main/java/org/apache/nemo/client/beam/NemoRunner.java
+++ b/client/src/main/java/org/apache/nemo/client/beam/NemoRunner.java
@@ -18,12 +18,12 @@
*/
package org.apache.nemo.client.beam;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.nemo.client.JobLauncher;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
+import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.compiler.frontend.beam.NemoPipelineOptions;
import org.apache.nemo.compiler.frontend.beam.PipelineVisitor;
@@ -35,6 +35,7 @@ public final class NemoRunner extends PipelineRunner<NemoPipelineResult> {
/**
* BEAM Pipeline Runner.
+ *
* @param nemoPipelineOptions PipelineOptions.
*/
private NemoRunner(final NemoPipelineOptions nemoPipelineOptions) {
@@ -64,6 +65,7 @@ public final class NemoRunner extends PipelineRunner<NemoPipelineResult> {
/**
* Static initializer for creating PipelineRunner with the given options.
+ *
* @param options given PipelineOptions.
* @return The created PipelineRunner.
*/
@@ -74,6 +76,7 @@ public final class NemoRunner extends PipelineRunner<NemoPipelineResult> {
/**
* Method to run the Pipeline.
+ *
* @param pipeline the Pipeline to run.
* @return The result of the pipeline.
*/
diff --git a/client/src/main/java/org/apache/nemo/client/beam/NemoRunnerRegistrar.java b/client/src/main/java/org/apache/nemo/client/beam/NemoRunnerRegistrar.java
index 832bcd6..a6b7f84 100644
--- a/client/src/main/java/org/apache/nemo/client/beam/NemoRunnerRegistrar.java
+++ b/client/src/main/java/org/apache/nemo/client/beam/NemoRunnerRegistrar.java
@@ -28,7 +28,7 @@ import org.apache.nemo.compiler.frontend.beam.NemoPipelineOptions;
/**
* Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the {@link NemoRunner}.
- *
+ * <p>
* {@link AutoService} will register Nemo's implementations of the {@link PipelineRunner} and {@link PipelineOptions}
* as available pipeline runner services.
*/
diff --git a/client/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java b/client/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
index 9b53d8c..86682f7 100644
--- a/client/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
+++ b/client/src/main/java/org/apache/reef/runtime/yarn/ClassPathBuilder.java
@@ -45,8 +45,8 @@ final class ClassPathBuilder {
*/
private static boolean couldBeYarnConfigurationPath(final String path) {
return path.contains("conf") ||
- path.contains("etc") ||
- path.contains(HadoopEnvironment.HADOOP_CONF_DIR);
+ path.contains("etc") ||
+ path.contains(HadoopEnvironment.HADOOP_CONF_DIR);
}
/**
diff --git a/client/src/test/java/org/apache/nemo/client/ClientDriverRPCTest.java b/client/src/test/java/org/apache/nemo/client/ClientDriverRPCTest.java
index ef2939f..24522a8 100644
--- a/client/src/test/java/org/apache/nemo/client/ClientDriverRPCTest.java
+++ b/client/src/test/java/org/apache/nemo/client/ClientDriverRPCTest.java
@@ -18,8 +18,8 @@
*/
package org.apache.nemo.client;
-import org.apache.nemo.runtime.master.ClientRPC;
import org.apache.nemo.runtime.common.comm.ControlMessage;
+import org.apache.nemo.runtime.master.ClientRPC;
import org.apache.reef.tang.Injector;
import org.apache.reef.tang.Tang;
import org.apache.reef.tang.exceptions.InjectionException;
@@ -35,6 +35,7 @@ import java.util.concurrent.CountDownLatch;
public final class ClientDriverRPCTest {
private DriverRPCServer driverRPCServer;
private ClientRPC clientRPC;
+
@Before
public void setupDriverRPCServer() {
// Initialize DriverRPCServer.
@@ -55,6 +56,7 @@ public final class ClientDriverRPCTest {
/**
* Test with empty set of handlers.
+ *
* @throws InjectionException on Exceptions on creating {@link ClientRPC}.
*/
@Test
@@ -64,37 +66,39 @@ public final class ClientDriverRPCTest {
/**
* Test with basic request method from driver to client.
- * @throws InjectionException on Exceptions on creating {@link ClientRPC}.
+ *
+ * @throws InjectionException on Exceptions on creating {@link ClientRPC}.
* @throws InterruptedException when interrupted while waiting EventHandler invocation
*/
@Test
public void testDriverToClientMethodInvocation() throws InjectionException, InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
driverRPCServer.registerHandler(ControlMessage.DriverToClientMessageType.DriverStarted,
- msg -> latch.countDown());
+ msg -> latch.countDown());
setupClientRPC();
clientRPC.send(ControlMessage.DriverToClientMessage.newBuilder()
- .setType(ControlMessage.DriverToClientMessageType.DriverStarted).build());
+ .setType(ControlMessage.DriverToClientMessageType.DriverStarted).build());
latch.await();
}
/**
* Test with request-response RPC between client and driver.
- * @throws InjectionException on Exceptions on creating {@link ClientRPC}.
+ *
+ * @throws InjectionException on Exceptions on creating {@link ClientRPC}.
* @throws InterruptedException when interrupted while waiting EventHandler invocation
*/
@Test
public void testBetweenClientAndDriver() throws InjectionException, InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
driverRPCServer.registerHandler(ControlMessage.DriverToClientMessageType.DriverStarted,
- msg -> driverRPCServer.send(ControlMessage.ClientToDriverMessage.newBuilder()
- .setType(ControlMessage.ClientToDriverMessageType.LaunchDAG)
- .setLaunchDAG(ControlMessage.LaunchDAGMessage.newBuilder().setDag("").build())
- .build()));
+ msg -> driverRPCServer.send(ControlMessage.ClientToDriverMessage.newBuilder()
+ .setType(ControlMessage.ClientToDriverMessageType.LaunchDAG)
+ .setLaunchDAG(ControlMessage.LaunchDAGMessage.newBuilder().setDag("").build())
+ .build()));
setupClientRPC();
clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.LaunchDAG, msg -> latch.countDown());
clientRPC.send(ControlMessage.DriverToClientMessage.newBuilder()
- .setType(ControlMessage.DriverToClientMessageType.DriverStarted).build());
+ .setType(ControlMessage.DriverToClientMessageType.DriverStarted).build());
latch.await();
}
}
diff --git a/client/src/test/java/org/apache/nemo/client/ClientEndpointTest.java b/client/src/test/java/org/apache/nemo/client/ClientEndpointTest.java
index 7095f39..2e4d1b9 100644
--- a/client/src/test/java/org/apache/nemo/client/ClientEndpointTest.java
+++ b/client/src/test/java/org/apache/nemo/client/ClientEndpointTest.java
@@ -19,11 +19,11 @@
package org.apache.nemo.client;
import org.apache.nemo.runtime.common.plan.PhysicalPlan;
+import org.apache.nemo.runtime.common.plan.TestPlanGenerator;
import org.apache.nemo.runtime.common.state.PlanState;
import org.apache.nemo.runtime.common.state.TaskState;
-import org.apache.nemo.runtime.master.metric.MetricMessageHandler;
import org.apache.nemo.runtime.master.PlanStateManager;
-import org.apache.nemo.runtime.common.plan.TestPlanGenerator;
+import org.apache.nemo.runtime.master.metric.MetricMessageHandler;
import org.apache.reef.tang.Injector;
import org.apache.reef.tang.Tang;
import org.junit.Test;
@@ -59,7 +59,7 @@ public class ClientEndpointTest {
// Create a PlanStateManager of a dag and create a DriverEndpoint with it.
final PhysicalPlan physicalPlan =
- TestPlanGenerator.generatePhysicalPlan(TestPlanGenerator.PlanType.TwoVerticesJoined, false);
+ TestPlanGenerator.generatePhysicalPlan(TestPlanGenerator.PlanType.TwoVerticesJoined, false);
final Injector injector = Tang.Factory.getTang().newInjector();
injector.bindVolatileInstance(MetricMessageHandler.class, mock(MetricMessageHandler.class));
final PlanStateManager planStateManager = injector.getInstance(PlanStateManager.class);
@@ -75,8 +75,8 @@ public class ClientEndpointTest {
// Check finish.
final List<String> tasks = physicalPlan.getStageDAG().getTopologicalSort().stream()
- .flatMap(stage -> planStateManager.getTaskAttemptsToSchedule(stage.getId()).stream())
- .collect(Collectors.toList());
+ .flatMap(stage -> planStateManager.getTaskAttemptsToSchedule(stage.getId()).stream())
+ .collect(Collectors.toList());
tasks.forEach(taskId -> planStateManager.onTaskStateChanged(taskId, TaskState.State.EXECUTING));
tasks.forEach(taskId -> planStateManager.onTaskStateChanged(taskId, TaskState.State.COMPLETE));
assertEquals(PlanState.State.COMPLETE, clientEndpoint.waitUntilJobFinish());
diff --git a/common/pom.xml b/common/pom.xml
index e35b27f..a325148 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -17,41 +17,42 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-project</artifactId>
- <version>0.2-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-project</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
- <artifactId>nemo-common</artifactId>
- <name>Nemo Common</name>
+ <artifactId>nemo-common</artifactId>
+ <name>Nemo Common</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.reef</groupId>
- <artifactId>reef-io</artifactId>
- <version>${reef.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.reef</groupId>
- <artifactId>reef-common</artifactId>
- <version>${reef.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-core</artifactId>
- <version>${beam.version}</version>
- </dependency>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.reef</groupId>
+ <artifactId>reef-io</artifactId>
+ <version>${reef.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.reef</groupId>
+ <artifactId>reef-common</artifactId>
+ <version>${reef.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <version>${beam.version}</version>
+ </dependency>
- </dependencies>
+ </dependencies>
</project>
diff --git a/common/src/main/java/org/apache/nemo/common/DirectByteArrayOutputStream.java b/common/src/main/java/org/apache/nemo/common/DirectByteArrayOutputStream.java
index 84b9e3e..9f41e5d 100644
--- a/common/src/main/java/org/apache/nemo/common/DirectByteArrayOutputStream.java
+++ b/common/src/main/java/org/apache/nemo/common/DirectByteArrayOutputStream.java
@@ -36,6 +36,7 @@ public final class DirectByteArrayOutputStream extends ByteArrayOutputStream {
/**
* Constructor specifying the size.
+ *
* @param size the initial size.
*/
public DirectByteArrayOutputStream(final int size) {
@@ -45,6 +46,7 @@ public final class DirectByteArrayOutputStream extends ByteArrayOutputStream {
/**
* Note that serializedBytes include invalid bytes.
* So we have to use it with the actualLength by using size() whenever needed.
+ *
* @return the buffer where data is stored.
*/
public byte[] getBufDirectly() {
diff --git a/common/src/main/java/org/apache/nemo/common/HashRange.java b/common/src/main/java/org/apache/nemo/common/HashRange.java
index edcb39b..c3209a4 100644
--- a/common/src/main/java/org/apache/nemo/common/HashRange.java
+++ b/common/src/main/java/org/apache/nemo/common/HashRange.java
@@ -30,8 +30,9 @@ public final class HashRange implements KeyRange<Integer> {
/**
* Private constructor.
+ *
* @param rangeBeginInclusive point at which the hash range starts (inclusive).
- * @param rangeEndExclusive point at which the hash range ends (exclusive).
+ * @param rangeEndExclusive point at which the hash range ends (exclusive).
*/
private HashRange(final int rangeBeginInclusive, final int rangeEndExclusive) {
if (rangeBeginInclusive < 0 || rangeEndExclusive < 0) {
@@ -121,7 +122,7 @@ public final class HashRange implements KeyRange<Integer> {
*/
@Override
public int hashCode() {
- return Arrays.hashCode(new Object[] {
+ return Arrays.hashCode(new Object[]{
rangeBeginInclusive,
rangeEndExclusive,
});
diff --git a/common/src/main/java/org/apache/nemo/common/KeyRange.java b/common/src/main/java/org/apache/nemo/common/KeyRange.java
index 8a66050..1a6b764 100644
--- a/common/src/main/java/org/apache/nemo/common/KeyRange.java
+++ b/common/src/main/java/org/apache/nemo/common/KeyRange.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
/**
* Represents the key range of data partitions within a block.
+ *
* @param <K> the type of key to assign for each partition.
*/
public interface KeyRange<K extends Serializable> extends Serializable {
diff --git a/common/src/main/java/org/apache/nemo/common/Pair.java b/common/src/main/java/org/apache/nemo/common/Pair.java
index 7ddc7d4..d14203f 100644
--- a/common/src/main/java/org/apache/nemo/common/Pair.java
+++ b/common/src/main/java/org/apache/nemo/common/Pair.java
@@ -23,6 +23,7 @@ import java.util.Objects;
/**
* Pair class.
+ *
* @param <A> type of the left element.
* @param <B> type of the right element.
*/
@@ -32,7 +33,8 @@ public final class Pair<A, B> implements Serializable {
/**
* Private constructor for Pair class.
- * @param left left element.
+ *
+ * @param left left element.
* @param right right element.
*/
private Pair(final A left, final B right) {
@@ -46,6 +48,7 @@ public final class Pair<A, B> implements Serializable {
public A left() {
return left;
}
+
/**
* @return right element
*/
@@ -61,23 +64,24 @@ public final class Pair<A, B> implements Serializable {
@Override
public boolean equals(final Object obj) {
return obj instanceof Pair
- && Objects.equals(this.left, ((Pair) obj).left)
- && Objects.equals(this.right, ((Pair) obj).right);
+ && Objects.equals(this.left, ((Pair) obj).left)
+ && Objects.equals(this.right, ((Pair) obj).right);
}
@Override
public int hashCode() {
return this.left == null
- ? (this.right == null ? 0 : this.right.hashCode() + 1)
- : (this.right == null ? this.left.hashCode() + 2 : this.left.hashCode() * 17 + this.right.hashCode());
+ ? (this.right == null ? 0 : this.right.hashCode() + 1)
+ : (this.right == null ? this.left.hashCode() + 2 : this.left.hashCode() * 17 + this.right.hashCode());
}
/**
* Static initializer of the Pair class.
- * @param left left element.
+ *
+ * @param left left element.
* @param right right element.
- * @param <A> Type of the left element.
- * @param <B> Type of the right element.
+ * @param <A> Type of the left element.
+ * @param <B> Type of the right element.
* @return the newly created Pair.
*/
public static <A, B> Pair<A, B> of(final A left, final B right) {
diff --git a/common/src/main/java/org/apache/nemo/common/StateMachine.java b/common/src/main/java/org/apache/nemo/common/StateMachine.java
index 28740a9..bc76b61 100644
--- a/common/src/main/java/org/apache/nemo/common/StateMachine.java
+++ b/common/src/main/java/org/apache/nemo/common/StateMachine.java
@@ -36,7 +36,8 @@ public final class StateMachine {
/**
* Private constructor.
- * @param stateMap Map of state enum to the state.
+ *
+ * @param stateMap Map of state enum to the state.
* @param initialState initial state of the state machine.
*/
private StateMachine(final Map<Enum, State> stateMap, final Enum initialState) {
@@ -62,7 +63,7 @@ public final class StateMachine {
*
* @param state a state
* @throws IllegalStateTransitionException the state is unknown state, or the transition
- * from the current state to the specified state is illegal
+ * from the current state to the specified state is illegal
*/
public synchronized void setState(final Enum state) throws IllegalStateTransitionException {
if (!stateMap.containsKey(state)) {
@@ -83,11 +84,11 @@ public final class StateMachine {
* if the current state equals to the expected state.
*
* @param expectedCurrentState an expected state
- * @param state a state
+ * @param state a state
* @return {@code true} if successful. {@code false} indicates that
* the actual value was not equal to the expected value.
* @throws IllegalStateTransitionException if the state is unknown state, or the transition
- * from the current state to the specified state is illegal
+ * from the current state to the specified state is illegal
*/
public synchronized boolean compareAndSetState(final Enum expectedCurrentState,
final Enum state) throws IllegalStateTransitionException {
@@ -96,7 +97,7 @@ public final class StateMachine {
setState(state);
} else {
LOG.debug("The expected current state [" + expectedCurrentState
- + "] is different from the actual state [" + currentState.stateEnum + "]");
+ + "] is different from the actual state [" + currentState.stateEnum + "]");
}
return compared;
@@ -114,7 +115,7 @@ public final class StateMachine {
*/
private String getPossibleTransitionsFromCurrentState() {
final StringBuilder stringBuilder = new StringBuilder()
- .append("Possible transitions from the current state are").append('\n');
+ .append("Possible transitions from the current state are").append('\n');
for (final Transition transition : currentState.getAllTransitions()) {
stringBuilder.append(transition).append('\n');
@@ -128,7 +129,7 @@ public final class StateMachine {
final StringBuilder stringBuilder = new StringBuilder();
for (final State state : stateMap.values()) {
stringBuilder.append(state).append('\n')
- .append("Possible transitions:").append('\n');
+ .append("Possible transitions:").append('\n');
for (final Transition transition : state.getAllTransitions()) {
stringBuilder.append(transition).append('\n');
}
@@ -154,7 +155,8 @@ public final class StateMachine {
/**
* Private constructor.
- * @param stateEnum Enum of the state.
+ *
+ * @param stateEnum Enum of the state.
* @param description Description of the state.
*/
private State(final Enum stateEnum, final String description) {
@@ -165,6 +167,7 @@ public final class StateMachine {
/**
* Adds a transition to the state.
+ *
* @param transition the added transition.
*/
private void addTransition(final Transition transition) {
@@ -206,8 +209,9 @@ public final class StateMachine {
/**
* Private constructor.
- * @param from State from which we switch.
- * @param to State that we switch to.
+ *
+ * @param from State from which we switch.
+ * @param to State that we switch to.
* @param description Description of the transition.
*/
private Transition(final State from, final State to, final String description) {
@@ -244,7 +248,7 @@ public final class StateMachine {
/**
* Adds a state with name and description.
*
- * @param stateEnum enumeration indicating the state
+ * @param stateEnum enumeration indicating the state
* @param description description of the state
* @return the builder
* @throws RuntimeException if the state was already added
@@ -275,12 +279,12 @@ public final class StateMachine {
/**
* Adds a transition with description.
*
- * @param from from state name
- * @param to to state name
+ * @param from from state name
+ * @param to to state name
* @param description description of the transition
* @return the builder
* @throws RuntimeException if either from or to state was not added, or the same transition
- * was already added
+ * was already added
*/
public Builder addTransition(final Enum from, final Enum to, final String description) {
if (!stateEnumSet.contains(from)) {
diff --git a/common/src/main/java/org/apache/nemo/common/Util.java b/common/src/main/java/org/apache/nemo/common/Util.java
index 4d9e042..1156704 100644
--- a/common/src/main/java/org/apache/nemo/common/Util.java
+++ b/common/src/main/java/org/apache/nemo/common/Util.java
@@ -55,6 +55,7 @@ public final class Util {
/**
* Finds the project root path.
+ *
* @return the project root path.
*/
public static String fetchProjectRootPath() {
@@ -63,6 +64,7 @@ public final class Util {
/**
* Helper method to recursively find the LICENSE file.
+ *
* @param path the path to search for.
* @return the path containing the LICENSE file.
*/
@@ -102,8 +104,8 @@ public final class Util {
/**
* @param edgeToClone to copy execution properties from.
- * @param newSrc of the new edge.
- * @param newDst of the new edge.
+ * @param newSrc of the new edge.
+ * @param newDst of the new edge.
* @return the new edge.
*/
public static IREdge cloneEdge(final IREdge edgeToClone,
@@ -119,8 +121,8 @@ public final class Util {
*
* @param commPattern to use.
* @param edgeToClone to copy execution properties from.
- * @param newSrc of the new edge.
- * @param newDst of the new edge.
+ * @param newSrc of the new edge.
+ * @param newDst of the new edge.
* @return the new edge.
*/
public static IREdge cloneEdge(final CommunicationPatternProperty.Value commPattern,
@@ -204,6 +206,7 @@ public final class Util {
/**
* Method for the instrumentation: for getting the object size.
+ *
* @param args arguments.
* @param inst the instrumentation.
*/
@@ -213,6 +216,7 @@ public final class Util {
/**
* Get the object byte size.
+ *
* @param o object to measure.
* @return the bytes of the object.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/coder/BytesDecoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/BytesDecoderFactory.java
index 45c47b3..594d412 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/BytesDecoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/BytesDecoderFactory.java
@@ -43,6 +43,7 @@ public final class BytesDecoderFactory implements DecoderFactory<byte[]> {
/**
* Static initializer of the decoder.
+ *
* @return the initializer.
*/
public static BytesDecoderFactory of() {
@@ -70,7 +71,7 @@ public final class BytesDecoderFactory implements DecoderFactory<byte[]> {
/**
* Constructor.
*
- * @param inputStream the input stream to decode.
+ * @param inputStream the input stream to decode.
*/
private BytesDecoder(final InputStream inputStream) {
this.inputStream = inputStream;
diff --git a/common/src/main/java/org/apache/nemo/common/coder/BytesEncoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/BytesEncoderFactory.java
index 6666a7b..afe1623 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/BytesEncoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/BytesEncoderFactory.java
@@ -21,7 +21,8 @@ package org.apache.nemo.common.coder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.IOException;
+import java.io.OutputStream;
/**
* A {@link EncoderFactory} which is used for an array of bytes.
@@ -40,6 +41,7 @@ public final class BytesEncoderFactory implements EncoderFactory<byte[]> {
/**
* Static initializer of the encoder.
+ *
* @return the initializer.
*/
public static BytesEncoderFactory of() {
diff --git a/common/src/main/java/org/apache/nemo/common/coder/IntDecoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/IntDecoderFactory.java
index 41653f2..146965b 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/IntDecoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/IntDecoderFactory.java
@@ -18,7 +18,9 @@
*/
package org.apache.nemo.common.coder;
-import java.io.*;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
/**
* A {@link DecoderFactory} which is used for an integer.
@@ -36,6 +38,7 @@ public final class IntDecoderFactory implements DecoderFactory<Integer> {
/**
* Static initializer of the coder.
+ *
* @return the initializer.
*/
public static IntDecoderFactory of() {
@@ -62,7 +65,7 @@ public final class IntDecoderFactory implements DecoderFactory<Integer> {
/**
* Constructor.
*
- * @param inputStream the input stream to decode.
+ * @param inputStream the input stream to decode.
*/
private IntDecoder(final InputStream inputStream) {
// If the inputStream is closed well in upper level, it is okay to not close this stream
diff --git a/common/src/main/java/org/apache/nemo/common/coder/IntEncoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/IntEncoderFactory.java
index 6be865e..d47ee50 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/IntEncoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/IntEncoderFactory.java
@@ -18,7 +18,9 @@
*/
package org.apache.nemo.common.coder;
-import java.io.*;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
/**
* A {@link EncoderFactory} which is used for an integer.
@@ -36,6 +38,7 @@ public final class IntEncoderFactory implements EncoderFactory<Integer> {
/**
* Static initializer of the coder.
+ *
* @return the initializer.
*/
public static IntEncoderFactory of() {
diff --git a/common/src/main/java/org/apache/nemo/common/coder/LongDecoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/LongDecoderFactory.java
index 622ada2..0a0804c 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/LongDecoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/LongDecoderFactory.java
@@ -38,6 +38,7 @@ public final class LongDecoderFactory implements DecoderFactory<Long> {
/**
* Static initializer of the coder.
+ *
* @return the initializer.
*/
public static LongDecoderFactory of() {
@@ -63,7 +64,7 @@ public final class LongDecoderFactory implements DecoderFactory<Long> {
/**
* Constructor.
*
- * @param inputStream the input stream to decode.
+ * @param inputStream the input stream to decode.
*/
private LongDecoder(final InputStream inputStream) {
// If the inputStream is closed well in upper level, it is okay to not close this stream
diff --git a/common/src/main/java/org/apache/nemo/common/coder/LongEncoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/LongEncoderFactory.java
index d7c6f08..b4a3086 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/LongEncoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/LongEncoderFactory.java
@@ -38,6 +38,7 @@ public final class LongEncoderFactory implements EncoderFactory<Long> {
/**
* Static initializer of the coder.
+ *
* @return the initializer.
*/
public static LongEncoderFactory of() {
diff --git a/common/src/main/java/org/apache/nemo/common/coder/PairDecoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/PairDecoderFactory.java
index 790690d..1e36310 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/PairDecoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/PairDecoderFactory.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
/**
* An DecoderFactory for {@link Pair}. Reference: KvCoder in BEAM.
+ *
* @param <A> type for the left coder.
* @param <B> type for the right coder.
*/
@@ -49,8 +50,8 @@ public final class PairDecoderFactory<A, B> implements DecoderFactory<Pair<A, B>
*
* @param leftDecoderFactory left coder.
* @param rightDecoderFactory right coder.
- * @param <A> type of the left element.
- * @param <B> type of the right element.
+ * @param <A> type of the left element.
+ * @param <B> type of the right element.
* @return the new PairDecoderFactory.
*/
public static <A, B> PairDecoderFactory<A, B> of(final DecoderFactory<A> leftDecoderFactory,
@@ -76,6 +77,7 @@ public final class PairDecoderFactory<A, B> implements DecoderFactory<Pair<A, B>
/**
* PairDecoder.
+ *
* @param <T1> type for the left coder.
* @param <T2> type for the right coder.
*/
@@ -87,7 +89,7 @@ public final class PairDecoderFactory<A, B> implements DecoderFactory<Pair<A, B>
/**
* Constructor.
*
- * @param inputStream the input stream to decode.
+ * @param inputStream the input stream to decode.
* @param leftDecoderFactory the actual decoder to use for left elements.
* @param rightDecoderFactory the actual decoder to use for right elements.
* @throws IOException if fail to instantiate coders.
diff --git a/common/src/main/java/org/apache/nemo/common/coder/PairEncoderFactory.java b/common/src/main/java/org/apache/nemo/common/coder/PairEncoderFactory.java
index 030c336..b2db262 100644
--- a/common/src/main/java/org/apache/nemo/common/coder/PairEncoderFactory.java
+++ b/common/src/main/java/org/apache/nemo/common/coder/PairEncoderFactory.java
@@ -19,11 +19,13 @@
package org.apache.nemo.common.coder;
import org.apache.nemo.common.Pair;
+
import java.io.IOException;
import java.io.OutputStream;
/**
* An EncoderFactory for {@link Pair}. Reference: KvCoder in BEAM.
+ *
* @param <A> type for the left coder.
* @param <B> type for the right coder.
*/
@@ -48,8 +50,8 @@ public final class PairEncoderFactory<A, B> implements EncoderFactory<Pair<A, B>
*
* @param leftEncoderFactory left coder.
* @param rightEncoderFactory right coder.
- * @param <A> type of the left element.
- * @param <B> type of the right element.
+ * @param <A> type of the left element.
+ * @param <B> type of the right element.
* @return the new PairEncoderFactory.
*/
public static <A, B> PairEncoderFactory<A, B> of(final EncoderFactory<A> leftEncoderFactory,
@@ -75,6 +77,7 @@ public final class PairEncoderFactory<A, B> implements EncoderFactory<Pair<A, B>
/**
* PairEncoder.
+ *
* @param <T1> type for the left coder.
* @param <T2> type for the right coder.
*/
@@ -86,7 +89,7 @@ public final class PairEncoderFactory<A, B> implements EncoderFactory<Pair<A, B>
/**
* Constructor.
*
- * @param outputStream the output stream to store the encoded bytes.
+ * @param outputStream the output stream to store the encoded bytes.
* @param leftEncoderFactory the actual encoder to use for left elements.
* @param rightEncoderFactory the actual encoder to use for right elements.
* @throws IOException if fail to instantiate coders.
diff --git a/common/src/main/java/org/apache/nemo/common/dag/DAG.java b/common/src/main/java/org/apache/nemo/common/dag/DAG.java
index de11d98..bbe6042 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/DAG.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/DAG.java
@@ -37,6 +37,7 @@ import java.util.stream.Collectors;
/**
* DAG implementation.
+ *
* @param <V> the vertex type
* @param <E> the edge type
*/
@@ -52,11 +53,12 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
/**
* Constructor of DAG, called by the DAGBuilder.
- * @param vertices set of vertices.
- * @param incomingEdges map of incoming edges for each vertex.
- * @param outgoingEdges map of outgoing edges for each vertex.
+ *
+ * @param vertices set of vertices.
+ * @param incomingEdges map of incoming edges for each vertex.
+ * @param outgoingEdges map of outgoing edges for each vertex.
* @param assignedLoopVertexMap map of assignedLoopVertex info.
- * @param loopStackDepthMap map of stack depth of LoopVertices.
+ * @param loopStackDepthMap map of stack depth of LoopVertices.
*/
public DAG(final Set<V> vertices,
final Map<V, Set<E>> incomingEdges,
@@ -68,9 +70,9 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
this.outgoingEdges = new HashMap<>();
vertices.stream().sorted(Comparator.comparingInt(Vertex::getNumericId)).forEachOrdered(this.vertices::add);
incomingEdges.forEach((v, es) -> this.incomingEdges.put(v.getId(),
- es.stream().sorted(Comparator.comparingInt(Edge::getNumericId)).collect(Collectors.toList())));
+ es.stream().sorted(Comparator.comparingInt(Edge::getNumericId)).collect(Collectors.toList())));
outgoingEdges.forEach((v, es) -> this.outgoingEdges.put(v.getId(),
- es.stream().sorted(Comparator.comparingInt(Edge::getNumericId)).collect(Collectors.toList())));
+ es.stream().sorted(Comparator.comparingInt(Edge::getNumericId)).collect(Collectors.toList())));
this.rootVertices = new ArrayList<>();
vertices.forEach(v -> {
@@ -150,7 +152,7 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
}
}
throw new IllegalEdgeOperationException(
- new Throwable("There exists no edge from " + srcVertexId + " to " + dstVertexId));
+ new Throwable("There exists no edge from " + srcVertexId + " to " + dstVertexId));
}
@Override
@@ -169,8 +171,9 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
/**
* Recursively adds ancestors of a vertex to the given list.
+ *
* @param ancestorList to accumulate the ancestors.
- * @param vertexId to find the ancestors for.
+ * @param vertexId to find the ancestors for.
*/
private void addAncestors(final List<V> ancestorList, final String vertexId) {
getParents(vertexId).forEach(parent -> {
@@ -208,15 +211,15 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
public void dfsTraverse(final Consumer<V> function, final TraversalOrder traversalOrder) {
final Set<V> visited = new HashSet<>();
getVertices().stream().filter(vertex -> incomingEdges.get(vertex.getId()).isEmpty()) // root Operators
- .filter(vertex -> !visited.contains(vertex))
- .forEachOrdered(vertex -> dfsDo(vertex, function, traversalOrder, visited));
+ .filter(vertex -> !visited.contains(vertex))
+ .forEachOrdered(vertex -> dfsDo(vertex, function, traversalOrder, visited));
}
@Override
public void dfsDo(final V vertex,
- final Consumer<V> vertexConsumer,
- final TraversalOrder traversalOrder,
- final Set<V> visited) {
+ final Consumer<V> vertexConsumer,
+ final TraversalOrder traversalOrder,
+ final Set<V> visited) {
visited.add(vertex);
if (traversalOrder == TraversalOrder.PreOrder) {
vertexConsumer.accept(vertex);
@@ -224,8 +227,8 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
final List<E> outEdges = getOutgoingEdgesOf(vertex);
if (!outEdges.isEmpty()) {
outEdges.stream().map(Edge::getDst)
- .filter(outOperator -> !visited.contains(outOperator))
- .forEachOrdered(outOperator -> dfsDo(outOperator, vertexConsumer, traversalOrder, visited));
+ .filter(outOperator -> !visited.contains(outOperator))
+ .forEachOrdered(outOperator -> dfsDo(outOperator, vertexConsumer, traversalOrder, visited));
}
if (traversalOrder == TraversalOrder.PostOrder) {
vertexConsumer.accept(vertex);
@@ -237,8 +240,10 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
final Set<V> reachableFromV1 = new HashSet<>();
final Set<V> reachableFromV2 = new HashSet<>();
- this.dfsDo(v1, (v) -> { }, TraversalOrder.PostOrder, reachableFromV1);
- this.dfsDo(v2, (v) -> { }, TraversalOrder.PostOrder, reachableFromV2);
+ this.dfsDo(v1, (v) -> {
+ }, TraversalOrder.PostOrder, reachableFromV1);
+ this.dfsDo(v2, (v) -> {
+ }, TraversalOrder.PostOrder, reachableFromV2);
return reachableFromV1.contains(v2) || reachableFromV2.contains(v1);
}
@@ -306,10 +311,10 @@ public final class DAG<V extends Vertex, E extends Edge<V>> implements DAGInterf
printWriter.println(toString());
printWriter.close();
LOG.debug(String.format("DAG JSON for %s is saved at %s"
- + " (Use https://service.jangho.kr/nemo-dag/ to visualize it.)", description, file.getPath()));
+ + " (Use https://service.jangho.kr/nemo-dag/ to visualize it.)", description, file.getPath()));
} catch (IOException e) {
LOG.warn(String.format("Cannot store JSON representation of %s to %s: %s",
- description, file.getPath(), e.toString()));
+ description, file.getPath(), e.toString()));
}
}
}
diff --git a/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java b/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
index bd5cc95..1d6af69 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
@@ -19,8 +19,11 @@
package org.apache.nemo.common.dag;
import org.apache.nemo.common.exception.CompileTimeOptimizationException;
-import org.apache.nemo.common.ir.vertex.*;
import org.apache.nemo.common.exception.IllegalVertexOperationException;
+import org.apache.nemo.common.ir.vertex.IRVertex;
+import org.apache.nemo.common.ir.vertex.LoopVertex;
+import org.apache.nemo.common.ir.vertex.OperatorVertex;
+import org.apache.nemo.common.ir.vertex.SourceVertex;
import org.apache.nemo.common.ir.vertex.executionproperty.MessageIdVertexProperty;
import org.apache.nemo.common.ir.vertex.utility.MessageAggregatorVertex;
import org.apache.nemo.common.ir.vertex.utility.SamplingVertex;
@@ -34,6 +37,7 @@ import java.util.stream.Stream;
/**
* DAG Builder.
+ *
* @param <V> the vertex type.
* @param <E> the edge type.
*/
@@ -157,10 +161,10 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
} else {
this.buildWithoutSourceSinkCheck().storeJSON("debug", "errored_ir", "Errored IR");
throw new IllegalVertexOperationException("The DAG does not contain"
- + (vertices.contains(src) ? "" : " [source]") + (vertices.contains(dst) ? "" : " [destination]")
- + " of the edge: [" + (src == null ? null : src.getId())
- + "]->[" + (dst == null ? null : dst.getId()) + "] in "
- + vertices.stream().map(V::getId).collect(Collectors.toSet()));
+ + (vertices.contains(src) ? "" : " [source]") + (vertices.contains(dst) ? "" : " [destination]")
+ + " of the edge: [" + (src == null ? null : src.getId())
+ + "]->[" + (dst == null ? null : dst.getId()) + "] in "
+ + vertices.stream().map(V::getId).collect(Collectors.toSet()));
}
return this;
}
@@ -212,8 +216,8 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
throw getException("DAG contains a cycle", vertex.toString());
} else {
outgoingEdges.get(vertex).stream().map(Edge::getDst)
- .filter(v -> !visited.contains(v))
- .forEachOrdered(v -> cycleCheck(stack, visited, v));
+ .filter(v -> !visited.contains(v))
+ .forEachOrdered(v -> cycleCheck(stack, visited, v));
}
stack.pop();
}
@@ -224,14 +228,14 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
private void sourceCheck() {
// We observe IRVertex that do not have any incoming edges.
final Supplier<Stream<V>> verticesToObserve = () -> vertices.stream().filter(v -> incomingEdges.get(v).isEmpty())
- .filter(v -> v instanceof IRVertex);
+ .filter(v -> v instanceof IRVertex);
// They should all match SourceVertex
if (!(verticesToObserve.get().allMatch(v -> (v instanceof SourceVertex)
|| (v instanceof SamplingVertex && ((SamplingVertex) v).getCloneOfOriginalVertex() instanceof SourceVertex)))) {
final String problematicVertices = verticesToObserve.get()
- .filter(v -> !(v instanceof SourceVertex))
- .map(V::getId)
- .collect(Collectors.toList()).toString();
+ .filter(v -> !(v instanceof SourceVertex))
+ .map(V::getId)
+ .collect(Collectors.toList()).toString();
throw getException("DAG source check failed while building DAG", problematicVertices);
}
}
@@ -242,14 +246,14 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
private void sinkCheck() {
// We observe IRVertex that do not have any outgoing edges.
final Supplier<Stream<V>> verticesToObserve = () -> vertices.stream()
- .filter(v -> outgoingEdges.get(v).isEmpty())
- .filter(v -> v instanceof IRVertex);
+ .filter(v -> outgoingEdges.get(v).isEmpty())
+ .filter(v -> v instanceof IRVertex);
// They should either be OperatorVertex or LoopVertex
if (verticesToObserve.get().anyMatch(v ->
!(v instanceof OperatorVertex || v instanceof LoopVertex))) {
final String problematicVertices = verticesToObserve.get().filter(v ->
- !(v instanceof OperatorVertex || v instanceof LoopVertex))
- .map(V::getId).collect(Collectors.toList()).toString();
+ !(v instanceof OperatorVertex || v instanceof LoopVertex))
+ .map(V::getId).collect(Collectors.toList()).toString();
throw getException("DAG sink check failed while building DAG", problematicVertices);
}
}
@@ -285,7 +289,7 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
final Stack<V> stack = new Stack<>();
final Set<V> visited = new HashSet<>();
vertices.stream().filter(v -> incomingEdges.get(v).isEmpty()) // source operators
- .forEachOrdered(v -> cycleCheck(stack, visited, v));
+ .forEachOrdered(v -> cycleCheck(stack, visited, v));
}
if (source) {
sourceCheck();
@@ -332,7 +336,8 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
/**
* Generates a user-friendly exception message.
- * @param reason of the exception.
+ *
+ * @param reason of the exception.
* @param problematicObjects that caused the exception.
* @return exception object.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/dag/DAGInterface.java b/common/src/main/java/org/apache/nemo/common/dag/DAGInterface.java
index eba2e4a..7040a9c 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/DAGInterface.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/DAGInterface.java
@@ -23,7 +23,8 @@ import org.apache.nemo.common.exception.IllegalEdgeOperationException;
import org.apache.nemo.common.ir.vertex.LoopVertex;
import java.io.Serializable;
-import java.util.*;
+import java.util.List;
+import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -37,6 +38,7 @@ import java.util.function.Predicate;
public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Serializable {
/**
* Retrieves the vertex given its ID.
+ *
* @param id of the vertex to retrieve
* @return the vertex
*/
@@ -44,6 +46,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the vertices of this DAG.
+ *
* @return the list of vertices.
* Note that the result is never null, ensured by {@link DAGBuilder}.
*/
@@ -51,18 +54,21 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the edges of this DAG.
+ *
* @return the list of edges.
*/
List<E> getEdges();
/**
* Retrieves the root vertices of this DAG.
+ *
* @return the list of root vertices.
*/
List<V> getRootVertices();
/**
* Retrieves the incoming edges of the given vertex.
+ *
* @param v the subject vertex.
* @return the list of incoming edges to the vertex.
* Note that the result is never null, ensured by {@link DAGBuilder}.
@@ -71,6 +77,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the incoming edges of the given vertex.
+ *
* @param vertexId the ID of the subject vertex.
* @return the list of incoming edges to the vertex.
* Note that the result is never null, ensured by {@link DAGBuilder}.
@@ -79,6 +86,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the outgoing edges of the given vertex.
+ *
* @param v the subject vertex.
* @return the list of outgoing edges to the vertex.
* Note that the result is never null, ensured by {@link DAGBuilder}.
@@ -87,6 +95,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the outgoing edges of the given vertex.
+ *
* @param vertexId the ID of the subject vertex.
* @return the list of outgoing edges to the vertex.
* Note that the result is never null, ensured by {@link DAGBuilder}.
@@ -95,6 +104,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the parent vertices of the given vertex.
+ *
* @param vertexId the ID of the subject vertex.
* @return the list of parent vertices.
*/
@@ -102,6 +112,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the children vertices of the given vertex.
+ *
* @param vertexId the ID of the subject vertex.
* @return the list of children vertices.
*/
@@ -109,6 +120,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the edge between two vertices.
+ *
* @param srcVertexId the ID of the source vertex.
* @param dstVertexId the ID of the destination vertex.
* @return the edge if exists.
@@ -119,12 +131,14 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Gets the DAG's vertices in topologically sorted order.
* This function brings consistent results.
+ *
* @return the sorted list of vertices in topological order.
*/
List<V> getTopologicalSort();
/**
* Retrieves the ancestors of a vertex.
+ *
* @param vertexId to find the ancestors for.
* @return the list of ancestors.
*/
@@ -132,6 +146,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the descendants of a vertex.
+ *
* @param vertexId to find the descendants for.
* @return the list of descendants.
*/
@@ -139,6 +154,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Filters the vertices according to the given condition.
+ *
* @param condition that must be satisfied to be included in the filtered list.
* @return the list of vertices that meet the condition.
*/
@@ -147,6 +163,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Applies the function to each node in the DAG in a topological order.
* This function brings consistent results.
+ *
* @param function to apply.
*/
void topologicalDo(final Consumer<V> function);
@@ -161,17 +178,19 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Traverses the DAG by DFS, applying the given function.
- * @param function to apply.
+ *
+ * @param function to apply.
* @param traversalOrder which the DFS should be conducted.
*/
void dfsTraverse(final Consumer<V> function, final TraversalOrder traversalOrder);
/**
* A recursive helper function for {@link #dfsTraverse(Consumer, TraversalOrder)}.
- * @param vertex the root vertex of the remaining DAG.
+ *
+ * @param vertex the root vertex of the remaining DAG.
* @param vertexConsumer the function to apply.
* @param traversalOrder which the DFS should be conducted.
- * @param visited the set of nodes visited.
+ * @param visited the set of nodes visited.
*/
void dfsDo(final V vertex,
final Consumer<V> vertexConsumer,
@@ -180,6 +199,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Function checks whether there is a path between two vertices.
+ *
* @param v1 First vertex to check.
* @param v2 Second vertex to check.
* @return Whether or not there is a path between two vertices.
@@ -188,6 +208,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Checks whether the given vertex is assigned with a wrapping LoopVertex.
+ *
* @param v Vertex to check.
* @return whether or not it is wrapped by a LoopVertex
*/
@@ -195,6 +216,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the stack depth of the given vertex.
+ *
* @param v Vertex to check.
* @return The depth of the stack of LoopVertices for the vertex.
*/
@@ -202,6 +224,7 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Retrieves the wrapping LoopVertex of the vertex.
+ *
* @param v Vertex to check.
* @return The wrapping LoopVertex.
*/
@@ -214,8 +237,9 @@ public interface DAGInterface<V extends Vertex, E extends Edge<V>> extends Seria
/**
* Stores JSON representation of this DAG into a file.
- * @param directory the directory which JSON representation is saved to
- * @param name name of this DAG
+ *
+ * @param directory the directory which JSON representation is saved to
+ * @param name name of this DAG
* @param description description of this DAG
*/
void storeJSON(final String directory, final String name, final String description);
diff --git a/common/src/main/java/org/apache/nemo/common/dag/Edge.java b/common/src/main/java/org/apache/nemo/common/dag/Edge.java
index 32fddde..5c100bf 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/Edge.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/Edge.java
@@ -18,13 +18,13 @@
*/
package org.apache.nemo.common.dag;
-import java.io.Serializable;
-
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Serializable;
+
/**
* Connects two vertices of a DAG.
* This class can be extended for various DAG representations.
@@ -40,7 +40,8 @@ public class Edge<V extends Vertex> implements Serializable {
/**
* Constructor for Edge.
- * @param id ID of the edge.
+ *
+ * @param id ID of the edge.
* @param src source vertex.
* @param dst destination vertex.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/dag/Vertex.java b/common/src/main/java/org/apache/nemo/common/dag/Vertex.java
index 1596d4b..a6b07b7 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/Vertex.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/Vertex.java
@@ -42,6 +42,7 @@ public abstract class Vertex implements Serializable {
public final String getId() {
return id;
}
+
/**
* @return the numeric id of the vertex.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/eventhandler/CommonEventHandler.java b/common/src/main/java/org/apache/nemo/common/eventhandler/CommonEventHandler.java
index 515bdf0..14c6a36 100644
--- a/common/src/main/java/org/apache/nemo/common/eventhandler/CommonEventHandler.java
+++ b/common/src/main/java/org/apache/nemo/common/eventhandler/CommonEventHandler.java
@@ -22,11 +22,13 @@ import org.apache.reef.wake.EventHandler;
/**
* Class for handling common events.
+ *
* @param <T> type of the event to handle.
*/
public interface CommonEventHandler<T> extends EventHandler<T> {
/**
* fetches the class of the event.
+ *
* @return the class of the event.
*/
Class<T> getEventClass();
diff --git a/common/src/main/java/org/apache/nemo/common/eventhandler/CompilerEventHandler.java b/common/src/main/java/org/apache/nemo/common/eventhandler/CompilerEventHandler.java
index fccdb48..22ba3e6 100644
--- a/common/src/main/java/org/apache/nemo/common/eventhandler/CompilerEventHandler.java
+++ b/common/src/main/java/org/apache/nemo/common/eventhandler/CompilerEventHandler.java
@@ -20,6 +20,7 @@ package org.apache.nemo.common.eventhandler;
/**
* Class for handling events sent from Compiler.
+ *
* @param <T> type of the compiler event to handle.
*/
public interface CompilerEventHandler<T extends CompilerEvent> extends CommonEventHandler<T> {
diff --git a/common/src/main/java/org/apache/nemo/common/eventhandler/RuntimeEventHandler.java b/common/src/main/java/org/apache/nemo/common/eventhandler/RuntimeEventHandler.java
index 3cc28d9..8ac72f9 100644
--- a/common/src/main/java/org/apache/nemo/common/eventhandler/RuntimeEventHandler.java
+++ b/common/src/main/java/org/apache/nemo/common/eventhandler/RuntimeEventHandler.java
@@ -20,6 +20,7 @@ package org.apache.nemo.common.eventhandler;
/**
* Class for handling events sent from Runtime.
+ *
* @param <T> type of the runtime event to handle.
*/
public interface RuntimeEventHandler<T extends RuntimeEvent> extends CommonEventHandler<T> {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/ContainerException.java b/common/src/main/java/org/apache/nemo/common/exception/ContainerException.java
index c5a1cfc..02a355a 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/ContainerException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/ContainerException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class ContainerException extends RuntimeException {
/**
* ContainerException.
+ *
* @param cause cause
*/
public ContainerException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/IllegalEdgeOperationException.java b/common/src/main/java/org/apache/nemo/common/exception/IllegalEdgeOperationException.java
index 7b78c24..2c9585a 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/IllegalEdgeOperationException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/IllegalEdgeOperationException.java
@@ -27,6 +27,7 @@ package org.apache.nemo.common.exception;
public final class IllegalEdgeOperationException extends RuntimeException {
/**
* IllegalEdgeOperationException.
+ *
* @param cause cause
*/
public IllegalEdgeOperationException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/IllegalMessageException.java b/common/src/main/java/org/apache/nemo/common/exception/IllegalMessageException.java
index 3716a8c..54d0d53 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/IllegalMessageException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/IllegalMessageException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class IllegalMessageException extends RuntimeException {
/**
* IllegalMessageException.
+ *
* @param cause cause
*/
public IllegalMessageException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/IllegalStateTransitionException.java b/common/src/main/java/org/apache/nemo/common/exception/IllegalStateTransitionException.java
index d454361..6016588 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/IllegalStateTransitionException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/IllegalStateTransitionException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class IllegalStateTransitionException extends Exception {
/**
* IllegalStateTransitionException.
+ *
* @param cause cause
*/
public IllegalStateTransitionException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/IllegalVertexOperationException.java b/common/src/main/java/org/apache/nemo/common/exception/IllegalVertexOperationException.java
index cdb4e82..fb990d5 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/IllegalVertexOperationException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/IllegalVertexOperationException.java
@@ -26,6 +26,7 @@ package org.apache.nemo.common.exception;
public final class IllegalVertexOperationException extends RuntimeException {
/**
* IllegalVertexOperationException.
+ *
* @param message message
*/
public IllegalVertexOperationException(final String message) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/InvalidParameterException.java b/common/src/main/java/org/apache/nemo/common/exception/InvalidParameterException.java
index 70c7551..121c540 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/InvalidParameterException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/InvalidParameterException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class InvalidParameterException extends RuntimeException {
/**
* InvalidParameterException.
+ *
* @param message message
*/
public InvalidParameterException(final String message) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/JsonParseException.java b/common/src/main/java/org/apache/nemo/common/exception/JsonParseException.java
index f84b931..7047944 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/JsonParseException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/JsonParseException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class JsonParseException extends RuntimeException {
/**
* JsonParseException.
+ *
* @param cause cause
*/
public JsonParseException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/MetricException.java b/common/src/main/java/org/apache/nemo/common/exception/MetricException.java
index 93d46fc..0218423 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/MetricException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/MetricException.java
@@ -27,6 +27,7 @@ public class MetricException extends RuntimeException {
/**
* MetricException.
+ *
* @param cause the cause of the exception.
*/
public MetricException(final Throwable cause) {
@@ -35,6 +36,7 @@ public class MetricException extends RuntimeException {
/**
* MetricException.
+ *
* @param cause the cause of the exception.
*/
public MetricException(final String cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/NodeConnectionException.java b/common/src/main/java/org/apache/nemo/common/exception/NodeConnectionException.java
index 70e9f11..13e9b6d 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/NodeConnectionException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/NodeConnectionException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class NodeConnectionException extends RuntimeException {
/**
* NodeConnectionException.
+ *
* @param cause cause
*/
public NodeConnectionException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/PhysicalPlanGenerationException.java b/common/src/main/java/org/apache/nemo/common/exception/PhysicalPlanGenerationException.java
index 3e3eb09..7fdace1 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/PhysicalPlanGenerationException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/PhysicalPlanGenerationException.java
@@ -27,6 +27,7 @@ package org.apache.nemo.common.exception;
public final class PhysicalPlanGenerationException extends RuntimeException {
/**
* PhysicalPlanGenerationException.
+ *
* @param message message
*/
public PhysicalPlanGenerationException(final String message) {
@@ -35,6 +36,7 @@ public final class PhysicalPlanGenerationException extends RuntimeException {
/**
* PhysicalPlanGenerationException.
+ *
* @param e throwable cause of the exception.
*/
public PhysicalPlanGenerationException(final Throwable e) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/SchedulingException.java b/common/src/main/java/org/apache/nemo/common/exception/SchedulingException.java
index 40d2c20..72fea09 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/SchedulingException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/SchedulingException.java
@@ -26,6 +26,7 @@ package org.apache.nemo.common.exception;
public final class SchedulingException extends RuntimeException {
/**
* SchedulingException.
+ *
* @param exception exception
*/
public SchedulingException(final Throwable exception) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnknownExecutionStateException.java b/common/src/main/java/org/apache/nemo/common/exception/UnknownExecutionStateException.java
index da6a198..0c32dc9 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnknownExecutionStateException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnknownExecutionStateException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnknownExecutionStateException extends RuntimeException {
/**
* UnknownExecutionStateException.
+ *
* @param cause cause
*/
public UnknownExecutionStateException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnknownFailureCauseException.java b/common/src/main/java/org/apache/nemo/common/exception/UnknownFailureCauseException.java
index bbaa6ba..bf3b000 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnknownFailureCauseException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnknownFailureCauseException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnknownFailureCauseException extends RuntimeException {
/**
* UnknownFailureCauseException.
+ *
* @param cause cause
*/
public UnknownFailureCauseException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnrecoverableFailureException.java b/common/src/main/java/org/apache/nemo/common/exception/UnrecoverableFailureException.java
index 9c4ffbb..3f053ae 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnrecoverableFailureException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnrecoverableFailureException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnrecoverableFailureException extends RuntimeException {
/**
* UnrecoverableFailureException.
+ *
* @param cause cause
*/
public UnrecoverableFailureException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedBlockStoreException.java b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedBlockStoreException.java
index c45403e..106e0a6 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedBlockStoreException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedBlockStoreException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnsupportedBlockStoreException extends RuntimeException {
/**
* UnsupportedBlockStoreException.
+ *
* @param cause cause
*/
public UnsupportedBlockStoreException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedCommPatternException.java b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedCommPatternException.java
index d801090..32adee2 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedCommPatternException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedCommPatternException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnsupportedCommPatternException extends RuntimeException {
/**
* UnsupportedCommPatternException.
+ *
* @param cause cause
*/
public UnsupportedCommPatternException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedExecutionPropertyException.java b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedExecutionPropertyException.java
index 8647ec6..4abc646 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedExecutionPropertyException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedExecutionPropertyException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnsupportedExecutionPropertyException extends RuntimeException {
/**
* UnsupportedExecutionPropertyException.
+ *
* @param message message
*/
public UnsupportedExecutionPropertyException(final String message) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMethodException.java b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMethodException.java
index 7919a38..88b1091 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMethodException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMethodException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnsupportedMethodException extends RuntimeException {
/**
* UnsupportedMethodException.
+ *
* @param message message
*/
public UnsupportedMethodException(final String message) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMetricException.java b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMetricException.java
index 4870b82..9628b27 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMetricException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedMetricException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnsupportedMetricException extends RuntimeException {
/**
* UnsupportedMetricException.
+ *
* @param cause cause
*/
public UnsupportedMetricException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedPartitionerException.java b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedPartitionerException.java
index ec0db17..749795c 100644
--- a/common/src/main/java/org/apache/nemo/common/exception/UnsupportedPartitionerException.java
+++ b/common/src/main/java/org/apache/nemo/common/exception/UnsupportedPartitionerException.java
@@ -25,6 +25,7 @@ package org.apache.nemo.common.exception;
public final class UnsupportedPartitionerException extends RuntimeException {
/**
* UnsupportedPartitionerException.
+ *
* @param cause cause
*/
public UnsupportedPartitionerException(final Throwable cause) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/BoundedIteratorReadable.java b/common/src/main/java/org/apache/nemo/common/ir/BoundedIteratorReadable.java
index 586fc47..c1085fe 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/BoundedIteratorReadable.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/BoundedIteratorReadable.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
/**
* An abstract readable class that retrieves data from iterator.
+ *
* @param <O> output type.
*/
public abstract class BoundedIteratorReadable<O> implements Readable<O> {
@@ -30,6 +31,7 @@ public abstract class BoundedIteratorReadable<O> implements Readable<O> {
/**
* Initialize iterator.
+ *
* @return iterator
*/
protected abstract Iterator<O> initializeIterator();
diff --git a/common/src/main/java/org/apache/nemo/common/ir/IRDAG.java b/common/src/main/java/org/apache/nemo/common/ir/IRDAG.java
index 7e37817..29af0a2 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/IRDAG.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/IRDAG.java
@@ -55,12 +55,12 @@ import java.util.stream.Collectors;
* An IRDAG object captures a high-level data processing application (e.g., Spark/Beam application).
* - IRVertex: A data-parallel operation. (e.g., map)
* - IREdge: A data dependency between two operations. (e.g., shuffle)
- *
+ * <p>
* Largely two types of IRDAG optimization(modification) methods are provided.
* All of these methods preserve application semantics.
* - Annotation: setProperty(), getPropertyValue() on each IRVertex/IREdge
* - Reshaping: insert(), delete() on the IRDAG
- *
+ * <p>
* TODO #341: Rethink IRDAG insert() signatures
*/
@NotThreadSafe
@@ -98,6 +98,7 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Used internally by Nemo to advance the DAG snapshot after applying each pass.
+ *
* @param checker that compares the dagSnapshot and the modifiedDAG
* to determine if the snapshot can be set the current modifiedDAG.
* @return true if the checker passes, false otherwise.
@@ -121,7 +122,7 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Deletes a previously inserted utility vertex.
* (e.g., MessageBarrierVertex, StreamVertex, SamplingVertex)
- *
+ * <p>
* Notice that the actual number of vertices that will be deleted after this call returns can be more than one.
* We roll back the changes made with the previous insert(), while preserving application semantics.
*
@@ -155,13 +156,13 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Delete a group of vertex that corresponds to the specified vertex.
* And then recursively delete neighboring utility vertices.
- *
+ * <p>
* (WARNING) Only call this method inside delete(), or inside this method itself.
* This method uses buildWithoutSourceSinkCheck() for intermediate DAGs,
* which will be finally checked in delete().
*
* @param vertexToDelete to delete
- * @param visited vertex groups (because cyclic dependencies between vertex groups are possible)
+ * @param visited vertex groups (because cyclic dependencies between vertex groups are possible)
*/
private void deleteRecursively(final IRVertex vertexToDelete, final Set<IRVertex> visited) {
if (!Util.isUtilityVertex(vertexToDelete)) {
@@ -203,8 +204,9 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
modifiedDAG.getIncomingEdgesOf(vertexToDelete).stream()
.filter(e -> !Util.isControlEdge(e))
.map(IREdge::getSrc)
- .forEach(srcVertex-> { builder.connectVertices(
- Util.cloneEdge(streamVertexToOriginalEdge.get(vertexToDelete), srcVertex, dstVertex));
+ .forEach(srcVertex -> {
+ builder.connectVertices(
+ Util.cloneEdge(streamVertexToOriginalEdge.get(vertexToDelete), srcVertex, dstVertex));
});
});
modifiedDAG = builder.buildWithoutSourceSinkCheck();
@@ -237,14 +239,14 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Inserts a new vertex that streams data.
- *
+ * <p>
* Before: src - edgeToStreamize - dst
* After: src - edgeToStreamizeWithNewDestination - streamVertex - oneToOneEdge - dst
* (replaces the "Before" relationships)
- *
+ * <p>
* This preserves semantics as the streamVertex simply forwards data elements from the input edge to the output edge.
*
- * @param streamVertex to insert.
+ * @param streamVertex to insert.
* @param edgeToStreamize to modify.
*/
public void insert(final StreamVertex streamVertex, final IREdge edgeToStreamize) {
@@ -319,24 +321,24 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Inserts a new vertex that analyzes intermediate data, and triggers a dynamic optimization.
- *
+ * <p>
* For each edge in edgesToGetStatisticsOf...
- *
+ * <p>
* Before: src - edge - dst
* After: src - oneToOneEdge(a clone of edge) - messageBarrierVertex -
- * shuffleEdge - messageAggregatorVertex - broadcastEdge - dst
+ * shuffleEdge - messageAggregatorVertex - broadcastEdge - dst
* (the "Before" relationships are unmodified)
- *
+ * <p>
* This preserves semantics as the results of the inserted message vertices are never consumed by the original IRDAG.
- *
+ * <p>
* TODO #345: Simplify insert(MessageBarrierVertex)
*
- * @param messageBarrierVertex to insert.
+ * @param messageBarrierVertex to insert.
* @param messageAggregatorVertex to insert.
- * @param mbvOutputEncoder to use.
- * @param mbvOutputDecoder to use.
- * @param edgesToGetStatisticsOf to examine.
- * @param edgesToOptimize to optimize.
+ * @param mbvOutputEncoder to use.
+ * @param mbvOutputDecoder to use.
+ * @param edgesToGetStatisticsOf to examine.
+ * @param edgesToOptimize to optimize.
*/
public void insert(final MessageBarrierVertex messageBarrierVertex,
final MessageAggregatorVertex messageAggregatorVertex,
@@ -430,29 +432,29 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Inserts a set of samplingVertices that process sampled data.
- *
+ * <p>
* This method automatically inserts the following three types of edges.
* (1) Edges between samplingVertices to reflect the original relationship
* (2) Edges from the original IRDAG to samplingVertices that clone the inEdges of the original vertices
* (3) Edges from the samplingVertices to the original IRDAG to respect executeAfterSamplingVertices
- *
+ * <p>
* Suppose the caller supplies the following arguments to perform a "sampled run" of vertices {V1, V2},
* prior to executing them.
* - samplingVertices: {V1', V2'}
* - childrenOfSamplingVertices: {V1}
- *
+ * <p>
* Before: V1 - oneToOneEdge - V2 - shuffleEdge - V3
* After: V1' - oneToOneEdge - V2' - controlEdge - V1 - oneToOneEdge - V2 - shuffleEdge - V3
- *
+ * <p>
* This preserves semantics as the original IRDAG remains unchanged and unaffected.
- *
+ * <p>
* (Future calls to insert() can add new vertices that connect to sampling vertices. Such new vertices will also be
* wrapped with sampling vertices, as new vertices that consume outputs from sampling vertices will process
* a subset of data anyways, and no such new vertex will reach the original DAG except via control edges)
- *
+ * <p>
* TODO #343: Extend SamplingVertex control edges
*
- * @param toInsert sampling vertices.
+ * @param toInsert sampling vertices.
* @param executeAfter that must be executed after toInsert.
*/
public void insert(final Set<SamplingVertex> toInsert,
@@ -523,6 +525,7 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
/**
* Reshape unsafely, without guarantees on preserving application semantics.
* TODO #330: Refactor Unsafe Reshaping Passes
+ *
* @param unsafeReshapingFunction takes as input the underlying DAG, and outputs a reshaped DAG.
*/
public void reshapeUnsafely(final Function<DAG<IRVertex, IREdge>, DAG<IRVertex, IREdge>> unsafeReshapingFunction) {
@@ -567,8 +570,8 @@ public final class IRDAG implements DAGInterface<IRVertex, IREdge> {
}
/**
- * @param mbv src.
- * @param mav dst.
+ * @param mbv src.
+ * @param mav dst.
* @param encoder src-dst encoder.
* @param decoder src-dst decoder.
* @return the edge.
diff --git a/common/src/main/java/org/apache/nemo/common/ir/IdManager.java b/common/src/main/java/org/apache/nemo/common/ir/IdManager.java
index 389af8b..1aad410 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/IdManager.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/IdManager.java
@@ -53,7 +53,8 @@ public final class IdManager {
/**
* Save the vertex id for the vertices that can be cloned later on.
* WARN: this should guarantee that the vertex is no longer used, otherwise, it would result in duplicate IDs.
- * @param v the original vertex that is to be cloned later on (RootLoopVertex's vertex).
+ *
+ * @param v the original vertex that is to be cloned later on (RootLoopVertex's vertex).
* @param id The IDs of the identical vertices.
*/
public static void saveVertexId(final Vertex v, final String id) {
@@ -66,6 +67,7 @@ public final class IdManager {
* otherwise simply acts as the newVertexId method.
* WARN: the #saveVertexId method should no longer use the ID saved at that moment,
* in order for this method to work correctly.
+ *
* @param v the vertex to get the ID for.
* @return the ID for the vertex.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/OutputCollector.java b/common/src/main/java/org/apache/nemo/common/ir/OutputCollector.java
index 3fe333a..0865093 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/OutputCollector.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/OutputCollector.java
@@ -26,17 +26,20 @@ import java.io.Serializable;
* Interface through which Transform emits outputs.
* This is to be implemented in the runtime with
* runtime-specific distributed data movement and storage mechanisms.
+ *
* @param <O> output type.
*/
public interface OutputCollector<O> extends Serializable {
/**
* Single-destination emit.
+ *
* @param output value.
*/
void emit(O output);
/**
* Emit watermark to downstream vertices.
+ *
* @param watermark watermark
*/
void emitWatermark(Watermark watermark);
@@ -45,9 +48,10 @@ public interface OutputCollector<O> extends Serializable {
* Multi-destination emit.
* Currently unused, but might come in handy
* for operations like multi-output map.
+ *
* @param dstVertexId destination vertex id.
- * @param output value.
- * @param <T> output type.
+ * @param output value.
+ * @param <T> output type.
*/
<T> void emit(String dstVertexId, T output);
}
diff --git a/common/src/main/java/org/apache/nemo/common/ir/Readable.java b/common/src/main/java/org/apache/nemo/common/ir/Readable.java
index c18c9e1..63a6b54 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/Readable.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/Readable.java
@@ -25,6 +25,7 @@ import java.util.NoSuchElementException;
/**
* Interface for readable.
+ *
* @param <O> output type.
*/
public interface Readable<O> extends Serializable {
@@ -38,8 +39,9 @@ public interface Readable<O> extends Serializable {
* Method to read current data from the source.
* The caller should check whether the Readable is finished or not by using isFinished() method
* before calling this method.
- *
+ * <p>
* It can throw NoSuchElementException although it is not finished in Unbounded source.
+ *
* @return a data read by the readable.
* @throws NoSuchElementException when no element exists
*/
@@ -47,6 +49,7 @@ public interface Readable<O> extends Serializable {
/**
* Read watermark.
+ *
* @return watermark
*/
long readWatermark();
@@ -68,6 +71,7 @@ public interface Readable<O> extends Serializable {
/**
* Close.
+ *
* @throws IOException if file-based reader throws any.
*/
void close() throws IOException;
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/IREdge.java b/common/src/main/java/org/apache/nemo/common/ir/edge/IREdge.java
index 969fd5b..a7774fb 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/IREdge.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/IREdge.java
@@ -20,6 +20,7 @@ package org.apache.nemo.common.ir.edge;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.nemo.common.dag.Edge;
import org.apache.nemo.common.ir.IdManager;
import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
@@ -27,10 +28,8 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
import org.apache.nemo.common.ir.executionproperty.ExecutionPropertyMap;
import org.apache.nemo.common.ir.vertex.IRVertex;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
import java.io.Serializable;
-import java.util.*;
+import java.util.Optional;
/**
* Physical execution plan of intermediate data movement.
@@ -54,6 +53,7 @@ public final class IREdge extends Edge<IRVertex> {
/**
* Set an executionProperty of the IREdge.
+ *
* @param executionProperty the execution property.
* @return the IREdge with the execution property set.
*/
@@ -64,6 +64,7 @@ public final class IREdge extends Edge<IRVertex> {
/**
* Set an executionProperty of the IREdge, permanently.
+ *
* @param executionProperty the execution property.
* @return the IREdge with the execution property set.
*/
@@ -80,7 +81,7 @@ public final class IREdge extends Edge<IRVertex> {
* @return the execution property.
*/
public <T extends Serializable> Optional<T> getPropertyValue(
- final Class<? extends EdgeExecutionProperty<T>> executionPropertyKey) {
+ final Class<? extends EdgeExecutionProperty<T>> executionPropertyKey) {
return executionProperties.get(executionPropertyKey);
}
@@ -125,10 +126,10 @@ public final class IREdge extends Edge<IRVertex> {
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
- .append(getSrc().hashCode())
- .append(getDst().hashCode())
- .append(executionProperties)
- .toHashCode();
+ .append(getSrc().hashCode())
+ .append(getDst().hashCode())
+ .append(executionProperties)
+ .toHashCode();
}
@Override
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/AdditionalOutputTagProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/AdditionalOutputTagProperty.java
index 4c85cec..422fb00 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/AdditionalOutputTagProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/AdditionalOutputTagProperty.java
@@ -27,6 +27,7 @@ public final class AdditionalOutputTagProperty extends EdgeExecutionProperty<Str
/**
* Constructor.
+ *
* @param value tag id of additional input.
*/
private AdditionalOutputTagProperty(final String value) {
@@ -35,6 +36,7 @@ public final class AdditionalOutputTagProperty extends EdgeExecutionProperty<Str
/**
* Static method exposing constructor.
+ *
* @param value tag id of additional input.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/CommunicationPatternProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/CommunicationPatternProperty.java
index 7f35c81..a6971b9 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/CommunicationPatternProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/CommunicationPatternProperty.java
@@ -21,13 +21,15 @@ package org.apache.nemo.common.ir.edge.executionproperty;
import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
// TODO #492: modularizing runtime components for data communication pattern.
+
/**
* DataCommunicationPattern ExecutionProperty.
*/
public final class CommunicationPatternProperty
- extends EdgeExecutionProperty<CommunicationPatternProperty.Value> {
+ extends EdgeExecutionProperty<CommunicationPatternProperty.Value> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private CommunicationPatternProperty(final Value value) {
@@ -36,6 +38,7 @@ public final class CommunicationPatternProperty
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataFlowProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataFlowProperty.java
index eccb120..5c8de19 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataFlowProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataFlowProperty.java
@@ -26,6 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
public final class DataFlowProperty extends EdgeExecutionProperty<DataFlowProperty.Value> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private DataFlowProperty(final Value value) {
@@ -34,6 +35,7 @@ public final class DataFlowProperty extends EdgeExecutionProperty<DataFlowProper
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataPersistenceProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataPersistenceProperty.java
index b4bccee..51bbfcb 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataPersistenceProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataPersistenceProperty.java
@@ -26,6 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
public final class DataPersistenceProperty extends EdgeExecutionProperty<DataPersistenceProperty.Value> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private DataPersistenceProperty(final Value value) {
@@ -34,6 +35,7 @@ public final class DataPersistenceProperty extends EdgeExecutionProperty<DataPer
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataStoreProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataStoreProperty.java
index e53157b..c2dc191 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataStoreProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DataStoreProperty.java
@@ -26,6 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
public final class DataStoreProperty extends EdgeExecutionProperty<DataStoreProperty.Value> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private DataStoreProperty(final Value value) {
@@ -34,6 +35,7 @@ public final class DataStoreProperty extends EdgeExecutionProperty<DataStoreProp
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupProperty.java
index 61f3635..68facb8 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupProperty.java
@@ -27,6 +27,7 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
public final class DuplicateEdgeGroupProperty extends EdgeExecutionProperty<DuplicateEdgeGroupPropertyValue> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private DuplicateEdgeGroupProperty(final DuplicateEdgeGroupPropertyValue value) {
@@ -35,6 +36,7 @@ public final class DuplicateEdgeGroupProperty extends EdgeExecutionProperty<Dupl
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/MessageIdEdgeProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/MessageIdEdgeProperty.java
index 4a5d320..3878c9a 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/MessageIdEdgeProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/MessageIdEdgeProperty.java
@@ -28,6 +28,7 @@ import java.util.HashSet;
public final class MessageIdEdgeProperty extends EdgeExecutionProperty<HashSet<Integer>> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private MessageIdEdgeProperty(final HashSet<Integer> value) {
@@ -36,6 +37,7 @@ public final class MessageIdEdgeProperty extends EdgeExecutionProperty<HashSet<I
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionSetProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionSetProperty.java
index 15b291a..ec06555 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionSetProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionSetProperty.java
@@ -26,14 +26,14 @@ import java.util.ArrayList;
/**
* This property decides which partitions the tasks of the destination IRVertex should fetch.
* The position of a KeyRange in the list corresponds to the offset of the destination task.
- *
+ * <p>
* For example, in the following setup:
* Source IRVertex (Parallelism=2) - IREdge (Partitioner.Num=4) - Destination IRVertex (Parallelism=2)
- *
+ * <p>
* Setting PartitionSetProperty([0, 3), [3, 3)) on the IREdge with will enforce the following behaviors.
* - The first destination task fetches the first 3 partitions from each of the 2 data blocks
* - The second destination task fetches the last partitions from each of the 2 data blocks
- *
+ * <p>
* This property is useful for handling data skews.
* For example, if the size ratio of the 4 partitions in the above setup are (17%, 16%, 17%, 50%),
* then each of the destination task will evenly handle 50% of the load.
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionerProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionerProperty.java
index 78f321a..4da595a 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionerProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/PartitionerProperty.java
@@ -48,7 +48,7 @@ public final class PartitionerProperty
}
/**
- * @param type of the partitioner.
+ * @param type of the partitioner.
* @param numOfPartitions to create.
* @return the property.
*/
@@ -57,9 +57,9 @@ public final class PartitionerProperty
}
/**
- * @param type of the partitioner.
+ * @param type of the partitioner.
* @param numOfPartitions to create.
- * @param auto if the number of partitions is auto.
+ * @param auto if the number of partitions is auto.
* @return the property.
*/
private static PartitionerProperty of(final Type type, final int numOfPartitions, final boolean auto) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/EdgeExecutionProperty.java b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/EdgeExecutionProperty.java
index 6dfdfe9..0b8647c 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/EdgeExecutionProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/EdgeExecutionProperty.java
@@ -22,11 +22,13 @@ import java.io.Serializable;
/**
* {@link ExecutionProperty} for {@link org.apache.nemo.common.ir.edge.IREdge}.
+ *
* @param <T> Type of the value.
*/
public abstract class EdgeExecutionProperty<T extends Serializable> extends ExecutionProperty<T> {
/**
* Default constructor.
+ *
* @param value value of the EdgeExecutionProperty.
*/
public EdgeExecutionProperty(final T value) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionProperty.java b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionProperty.java
index 008d13c..a73034d 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionProperty.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
/**
* An abstract class for each execution factors.
+ *
* @param <T> Type of the value.
*/
public abstract class ExecutionProperty<T extends Serializable> implements Serializable {
@@ -29,6 +30,7 @@ public abstract class ExecutionProperty<T extends Serializable> implements Seria
/**
* Default constructor.
+ *
* @param value value of the ExecutionProperty.
*/
public ExecutionProperty(final T value) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMap.java b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMap.java
index 2d2e8ad..8a36b94 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMap.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMap.java
@@ -20,6 +20,8 @@ package org.apache.nemo.common.ir.executionproperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.nemo.common.coder.DecoderFactory;
import org.apache.nemo.common.coder.EncoderFactory;
import org.apache.nemo.common.exception.CompileTimeOptimizationException;
@@ -28,9 +30,6 @@ import org.apache.nemo.common.ir.edge.executionproperty.*;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.ir.vertex.executionproperty.ResourcePriorityProperty;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
import javax.annotation.concurrent.NotThreadSafe;
import java.io.Serializable;
import java.util.*;
@@ -40,6 +39,7 @@ import java.util.stream.Stream;
/**
* ExecutionPropertyMap Class, which uses HashMap for keeping track of ExecutionProperties for vertices and edges.
+ *
* @param <T> Type of {@link ExecutionProperty} this map stores.
*/
@NotThreadSafe
@@ -50,6 +50,7 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Constructor for ExecutionPropertyMap class.
+ *
* @param id ID of the vertex / edge to keep the execution property of.
*/
@VisibleForTesting
@@ -59,13 +60,14 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Static initializer for irEdges.
- * @param irEdge irEdge to keep the execution property of.
+ *
+ * @param irEdge irEdge to keep the execution property of.
* @param commPattern Data communication pattern type of the edge.
* @return The corresponding ExecutionPropertyMap.
*/
public static ExecutionPropertyMap<EdgeExecutionProperty> of(
- final IREdge irEdge,
- final CommunicationPatternProperty.Value commPattern) {
+ final IREdge irEdge,
+ final CommunicationPatternProperty.Value commPattern) {
final ExecutionPropertyMap<EdgeExecutionProperty> map = new ExecutionPropertyMap<>(irEdge.getId());
map.put(CommunicationPatternProperty.of(commPattern));
map.put(EncoderProperty.of(EncoderFactory.DUMMY_ENCODER_FACTORY));
@@ -94,6 +96,7 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Static initializer for irVertex.
+ *
* @param irVertex irVertex to keep the execution property of.
* @return The corresponding ExecutionPropertyMap.
*/
@@ -105,6 +108,7 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* ID of the item this ExecutionPropertyMap class is keeping track of.
+ *
* @return the ID of the item this ExecutionPropertyMap class is keeping track of.
*/
public String getId() {
@@ -113,6 +117,7 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Put the given execution property in the ExecutionPropertyMap. By default, it does not finalize the property.
+ *
* @param executionProperty execution property to insert.
* @return the previous execution property, or null if there was no execution property
* with the specified property key.
@@ -123,19 +128,20 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Put the given execution property in the ExecutionPropertyMap.
+ *
* @param executionProperty execution property to insert.
- * @param finalize whether or not to finalize the execution property.
+ * @param finalize whether or not to finalize the execution property.
* @return the previous execution property, or null if there was no execution property
* with the specified property key.
*/
public T put(final T executionProperty, final Boolean finalize) {
// check if the property has been already finalized. We don't mind overwriting an identical value.
if (finalizedProperties.contains(executionProperty.getClass())
- && properties.get(executionProperty.getClass()) != null
- && !properties.get(executionProperty.getClass()).equals(executionProperty)) {
+ && properties.get(executionProperty.getClass()) != null
+ && !properties.get(executionProperty.getClass()).equals(executionProperty)) {
throw new CompileTimeOptimizationException("Trying to overwrite a finalized execution property ["
- + executionProperty.getClass().getSimpleName() + "] from ["
- + properties.get(executionProperty.getClass()).getValue() + "] to [" + executionProperty.getValue() + "]");
+ + executionProperty.getClass().getSimpleName() + "] from ["
+ + properties.get(executionProperty.getClass()).getValue() + "] to [" + executionProperty.getValue() + "]");
}
// start the actual put process.
@@ -147,7 +153,8 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Get the value of the given execution property type.
- * @param <U> Type of the return value.
+ *
+ * @param <U> Type of the return value.
* @param executionPropertyKey the execution property type to find the value of.
* @return the value of the given execution property.
*/
@@ -158,6 +165,7 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* remove the execution property.
+ *
* @param key key of the execution property to remove.
* @return the removed execution property
*/
@@ -175,6 +183,7 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
/**
* Same as forEach function in Java 8, but for execution properties.
+ *
* @param action action to apply to each of the execution properties.
*/
public void forEachProperties(final Consumer<? super T> action) {
@@ -216,13 +225,13 @@ public final class ExecutionPropertyMap<T extends ExecutionProperty> implements
}
final ExecutionPropertyMap that = (ExecutionPropertyMap) obj;
return properties.values().stream().collect(Collectors.toSet())
- .equals(that.properties.values().stream().collect(Collectors.toSet()));
+ .equals(that.properties.values().stream().collect(Collectors.toSet()));
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
- .append(properties.values().stream().map(ExecutionProperty::getValue).collect(Collectors.toSet()))
- .toHashCode();
+ .append(properties.values().stream().map(ExecutionProperty::getValue).collect(Collectors.toSet()))
+ .toHashCode();
}
}
diff --git a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/VertexExecutionProperty.java b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/VertexExecutionProperty.java
index a5fabd8..aa04dc0 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/executionproperty/VertexExecutionProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/executionproperty/VertexExecutionProperty.java
@@ -22,11 +22,13 @@ import java.io.Serializable;
/**
* {@link ExecutionProperty} for {@link org.apache.nemo.common.ir.vertex.IRVertex}.
+ *
* @param <T> Type of the value.
*/
public abstract class VertexExecutionProperty<T extends Serializable> extends ExecutionProperty<T> {
/**
* Default constructor.
+ *
* @param value value of the VertexExecutionProperty.
*/
public VertexExecutionProperty(final T value) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/CachedSourceVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/CachedSourceVertex.java
index 331a635..d5ff4b5 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/CachedSourceVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/CachedSourceVertex.java
@@ -27,6 +27,7 @@ import java.util.List;
/**
* Bounded source vertex for cached data.
* It does not have actual data but just wraps the cached input data.
+ *
* @param <T> the type of data to emit.
*/
public final class CachedSourceVertex<T> extends SourceVertex<T> {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/IRVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/IRVertex.java
index 378fc40..4a3c6df 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/IRVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/IRVertex.java
@@ -20,11 +20,11 @@ package org.apache.nemo.common.ir.vertex;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.nemo.common.Cloneable;
+import org.apache.nemo.common.dag.Vertex;
import org.apache.nemo.common.ir.IdManager;
import org.apache.nemo.common.ir.executionproperty.ExecutionPropertyMap;
-import org.apache.nemo.common.dag.Vertex;
import org.apache.nemo.common.ir.executionproperty.VertexExecutionProperty;
-import org.apache.nemo.common.Cloneable;
import java.io.Serializable;
import java.util.Optional;
@@ -57,6 +57,7 @@ public abstract class IRVertex extends Vertex implements Cloneable<IRVertex> {
/**
* Static function to copy executionProperties from a vertex to the other.
+ *
* @param thatVertex the edge to copy executionProperties to.
*/
public final void copyExecutionPropertiesTo(final IRVertex thatVertex) {
@@ -87,12 +88,13 @@ public abstract class IRVertex extends Vertex implements Cloneable<IRVertex> {
/**
* Get the executionProperty of the IRVertex.
- * @param <T> Type of the return value.
+ *
+ * @param <T> Type of the return value.
* @param executionPropertyKey key of the execution property.
* @return the execution property.
*/
public final <T extends Serializable> Optional<T> getPropertyValue(
- final Class<? extends VertexExecutionProperty<T>> executionPropertyKey) {
+ final Class<? extends VertexExecutionProperty<T>> executionPropertyKey) {
return executionProperties.get(executionPropertyKey);
}
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/InMemorySourceVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/InMemorySourceVertex.java
index 43936d8..1cf4e7d 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/InMemorySourceVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/InMemorySourceVertex.java
@@ -18,9 +18,9 @@
*/
package org.apache.nemo.common.ir.vertex;
+import org.apache.nemo.common.Util;
import org.apache.nemo.common.ir.BoundedIteratorReadable;
import org.apache.nemo.common.ir.Readable;
-import org.apache.nemo.common.Util;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,6 +29,7 @@ import java.util.List;
/**
* Source vertex with the data in memory.
+ *
* @param <T> type of data.
*/
public final class InMemorySourceVertex<T> extends SourceVertex<T> {
@@ -100,6 +101,7 @@ public final class InMemorySourceVertex<T> extends SourceVertex<T> {
/**
* Simply returns the in-memory data.
+ *
* @param <T> type of the data.
*/
private static final class InMemorySourceReadable<T> extends BoundedIteratorReadable<T> {
@@ -108,6 +110,7 @@ public final class InMemorySourceVertex<T> extends SourceVertex<T> {
/**
* Constructor.
+ *
* @param initializedSourceData the source data.
*/
private InMemorySourceReadable(final Iterable<T> initializedSourceData) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
index b7a29e6..0f6ea2e 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
@@ -30,7 +30,10 @@ import org.apache.nemo.common.ir.edge.executionproperty.DuplicateEdgeGroupProper
import org.apache.nemo.common.ir.edge.executionproperty.DuplicateEdgeGroupPropertyValue;
import java.io.Serializable;
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import java.util.function.IntPredicate;
/**
@@ -118,7 +121,8 @@ public final class LoopVertex extends IRVertex {
/**
* Maps an edge from/to loop with the corresponding edge from/to internal vertex.
- * @param edgeWithLoop an edge from/to loop
+ *
+ * @param edgeWithLoop an edge from/to loop
* @param edgeWithInternalVertex the corresponding edge from/to internal vertex
*/
public void mapEdgeWithLoop(final IREdge edgeWithLoop, final IREdge edgeWithInternalVertex) {
@@ -136,6 +140,7 @@ public final class LoopVertex extends IRVertex {
/**
* Adds the incoming edge of the contained DAG.
+ *
* @param edge edge to add.
*/
public void addDagIncomingEdge(final IREdge edge) {
@@ -152,12 +157,14 @@ public final class LoopVertex extends IRVertex {
/**
* Adds an iterative incoming edge, from the previous iteration, but connection internally.
+ *
* @param edge edge to add.
*/
public void addIterativeIncomingEdge(final IREdge edge) {
this.iterativeIncomingEdges.putIfAbsent(edge.getDst(), new HashSet<>());
this.iterativeIncomingEdges.get(edge.getDst()).add(edge);
}
+
/**
* @return the iterative incoming edges inside the DAG.
*/
@@ -167,6 +174,7 @@ public final class LoopVertex extends IRVertex {
/**
* Adds a non-iterative incoming edge, from outside the previous iteration.
+ *
* @param edge edge to add.
*/
public void addNonIterativeIncomingEdge(final IREdge edge) {
@@ -183,12 +191,14 @@ public final class LoopVertex extends IRVertex {
/**
* Adds and outgoing edge of the contained DAG.
+ *
* @param edge edge to add.
*/
public void addDagOutgoingEdge(final IREdge edge) {
this.dagOutgoingEdges.putIfAbsent(edge.getSrc(), new HashSet<>());
this.dagOutgoingEdges.get(edge.getSrc()).add(edge);
}
+
/**
* @return outgoing edges of the contained DAG.
*/
@@ -202,13 +212,14 @@ public final class LoopVertex extends IRVertex {
public void markDuplicateEdges() {
nonIterativeIncomingEdges.forEach(((irVertex, irEdges) -> irEdges.forEach(irEdge -> {
irEdge.setProperty(
- DuplicateEdgeGroupProperty.of(new DuplicateEdgeGroupPropertyValue(String.valueOf(duplicateEdgeGroupId))));
+ DuplicateEdgeGroupProperty.of(new DuplicateEdgeGroupPropertyValue(String.valueOf(duplicateEdgeGroupId))));
duplicateEdgeGroupId++;
})));
}
/**
* Method for unrolling an iteration of the LoopVertex.
+ *
* @param dagBuilder DAGBuilder to add the unrolled iteration to.
* @return a LoopVertex with one less maximum iteration.
*/
@@ -236,7 +247,7 @@ public final class LoopVertex extends IRVertex {
// process DAG incoming edges.
getDagIncomingEdges().forEach((dstVertex, irEdges) -> irEdges.forEach(edge -> {
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- edge.getSrc(), originalToNewIRVertex.get(dstVertex));
+ edge.getSrc(), originalToNewIRVertex.get(dstVertex));
edge.copyExecutionPropertiesTo(newIrEdge);
dagBuilder.connectVertices(newIrEdge);
}));
@@ -245,7 +256,7 @@ public final class LoopVertex extends IRVertex {
// if termination condition met, we process the DAG outgoing edge.
getDagOutgoingEdges().forEach((srcVertex, irEdges) -> irEdges.forEach(edge -> {
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- originalToNewIRVertex.get(srcVertex), edge.getDst());
+ originalToNewIRVertex.get(srcVertex), edge.getDst());
edge.copyExecutionPropertiesTo(newIrEdge);
dagBuilder.addVertex(edge.getDst()).connectVertices(newIrEdge);
}));
@@ -256,7 +267,7 @@ public final class LoopVertex extends IRVertex {
this.nonIterativeIncomingEdges.forEach((dstVertex, irEdges) -> irEdges.forEach(this::addDagIncomingEdge));
this.iterativeIncomingEdges.forEach((dstVertex, irEdges) -> irEdges.forEach(edge -> {
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- originalToNewIRVertex.get(edge.getSrc()), dstVertex);
+ originalToNewIRVertex.get(edge.getSrc()), dstVertex);
edge.copyExecutionPropertiesTo(newIrEdge);
this.addDagIncomingEdge(newIrEdge);
}));
@@ -270,6 +281,7 @@ public final class LoopVertex extends IRVertex {
public Boolean loopTerminationConditionMet() {
return loopTerminationConditionMet(maxNumberOfIterations);
}
+
/**
* @param intPredicateInput input for the intPredicate of the loop termination condition.
* @return whether or not the loop termination condition has been met.
@@ -280,29 +292,34 @@ public final class LoopVertex extends IRVertex {
/**
* Set the maximum number of iterations.
+ *
* @param maxNum maximum number of iterations.
*/
public void setMaxNumberOfIterations(final Integer maxNum) {
this.maxNumberOfIterations = maxNum;
}
+
/**
* @return termination condition int predicate.
*/
public IntPredicate getTerminationCondition() {
return terminationCondition;
}
+
/**
* @return maximum number of iterations.
*/
public Integer getMaxNumberOfIterations() {
return this.maxNumberOfIterations;
}
+
/**
* increase the value of maximum number of iterations by 1.
*/
public void increaseMaxNumberOfIterations() {
this.maxNumberOfIterations++;
}
+
/**
* decrease the value of maximum number of iterations by 1.
*/
@@ -312,13 +329,14 @@ public final class LoopVertex extends IRVertex {
/**
* Check termination condition.
+ *
* @param that another vertex.
* @return true if equals.
*/
public boolean terminationConditionEquals(final LoopVertex that) {
if (this.maxNumberOfIterations.equals(that.getMaxNumberOfIterations()) && Util
- .checkEqualityOfIntPredicates(this.terminationCondition, that.getTerminationCondition(),
- this.maxNumberOfIterations)) {
+ .checkEqualityOfIntPredicates(this.terminationCondition, that.getTerminationCondition(),
+ this.maxNumberOfIterations)) {
return true;
}
return false;
@@ -326,6 +344,7 @@ public final class LoopVertex extends IRVertex {
/**
* Set the intPredicate termination condition for the LoopVertex.
+ *
* @param terminationCondition the termination condition to set.
*/
public void setTerminationCondition(final IntPredicate terminationCondition) {
@@ -347,6 +366,7 @@ public final class LoopVertex extends IRVertex {
/**
* Convert the crossing edges to JSON.
+ *
* @param map map of the crossing edges.
* @return a string of JSON showing the crossing edges.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/OperatorVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/OperatorVertex.java
index 4be72f0..1b2136d 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/OperatorVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/OperatorVertex.java
@@ -30,6 +30,7 @@ public class OperatorVertex extends IRVertex {
/**
* Constructor of OperatorVertex.
+ *
* @param t transform for the OperatorVertex.
*/
public OperatorVertex(final Transform t) {
@@ -39,6 +40,7 @@ public class OperatorVertex extends IRVertex {
/**
* Copy Constructor of OperatorVertex.
+ *
* @param that the source object for copying
*/
private OperatorVertex(final OperatorVertex that) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/SourceVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/SourceVertex.java
index f606cf5..490c240 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/SourceVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/SourceVertex.java
@@ -25,6 +25,7 @@ import java.util.List;
/**
* IRVertex that reads data from an external source.
* It is to be implemented in the compiler frontend with source-specific data fetching logic.
+ *
* @param <O> output type.
*/
public abstract class SourceVertex<O> extends IRVertex {
@@ -60,6 +61,7 @@ public abstract class SourceVertex<O> extends IRVertex {
/**
* Gets the estimated size of bytes. Returns 0L if not applicable.
+ *
* @return size of input bytes.
*/
public abstract long getEstimatedSizeBytes();
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ClonedSchedulingProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ClonedSchedulingProperty.java
index edce1d7..ee08619 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ClonedSchedulingProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ClonedSchedulingProperty.java
@@ -26,13 +26,14 @@ import java.io.Serializable;
/**
* Specifies cloned execution of a vertex.
- *
+ * <p>
* A major limitations of the current implementation:
* *ALL* of the clones are always scheduled immediately
*/
public final class ClonedSchedulingProperty extends VertexExecutionProperty<ClonedSchedulingProperty.CloneConf> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private ClonedSchedulingProperty(final CloneConf value) {
@@ -41,6 +42,7 @@ public final class ClonedSchedulingProperty extends VertexExecutionProperty<Clon
/**
* Static method exposing the constructor.
+ *
* @param conf value of the new execution property.
* @return the newly created execution property.
*/
@@ -74,7 +76,8 @@ public final class ClonedSchedulingProperty extends VertexExecutionProperty<Clon
/**
* Clone stragglers judiciously.
- * @param fractionToWaitFor before trying to clone.
+ *
+ * @param fractionToWaitFor before trying to clone.
* @param medianTimeMultiplier to identify stragglers.
*/
public CloneConf(final double fractionToWaitFor, final double medianTimeMultiplier) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/MessageIdVertexProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/MessageIdVertexProperty.java
index 9ba842f..60346f3 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/MessageIdVertexProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/MessageIdVertexProperty.java
@@ -26,6 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.VertexExecutionProperty;
public final class MessageIdVertexProperty extends VertexExecutionProperty<Integer> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private MessageIdVertexProperty(final Integer value) {
@@ -34,6 +35,7 @@ public final class MessageIdVertexProperty extends VertexExecutionProperty<Integ
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ParallelismProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ParallelismProperty.java
index dcfd4b1..992ebe2 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ParallelismProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ParallelismProperty.java
@@ -22,20 +22,20 @@ import org.apache.nemo.common.ir.executionproperty.VertexExecutionProperty;
/**
* This property decides the number of parallel tasks to use for executing the corresponding IRVertex.
- *
+ * <p>
* Changing the parallelism requires also changing other execution properties that refer to task offsets.
* Such execution properties include:
* {@link ResourceSiteProperty}
* {@link ResourceAntiAffinityProperty}
* {@link org.apache.nemo.common.ir.edge.executionproperty.PartitionerProperty}
* {@link org.apache.nemo.common.ir.edge.executionproperty.PartitionSetProperty}
- *
+ * <p>
* Moreover, vertices with one-to-one relationships must have the same parallelism.
* {@link org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty}
- *
+ * <p>
* Finally, the parallelism cannot be larger than the number of source (e.g., HDFS) input data partitions.
* {@link org.apache.nemo.common.ir.vertex.SourceVertex}
- *
+ * <p>
* A violation of any of the above criteria will be caught by Nemo, to ensure correct application semantics.
*/
public final class ParallelismProperty extends VertexExecutionProperty<Integer> {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourcePriorityProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourcePriorityProperty.java
index 798ea54..7b6c6b3 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourcePriorityProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourcePriorityProperty.java
@@ -26,6 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.VertexExecutionProperty;
public final class ResourcePriorityProperty extends VertexExecutionProperty<String> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private ResourcePriorityProperty(final String value) {
@@ -34,6 +35,7 @@ public final class ResourcePriorityProperty extends VertexExecutionProperty<Stri
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSiteProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSiteProperty.java
index 4e44154..ee28e1b 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSiteProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSiteProperty.java
@@ -27,20 +27,22 @@ import java.util.HashMap;
* TODO #169: Use sites (not node names) in ResourceSiteProperty
*/
public final class ResourceSiteProperty extends VertexExecutionProperty<HashMap<String, Integer>> {
- /**
- * Default constructor.
- * @param value the map from location to the number of Task that must be executed on the node
- */
- public ResourceSiteProperty(final HashMap<String, Integer> value) {
- super(value);
- }
+ /**
+ * Default constructor.
+ *
+ * @param value the map from location to the number of Task that must be executed on the node
+ */
+ public ResourceSiteProperty(final HashMap<String, Integer> value) {
+ super(value);
+ }
- /**
- * Static method for constructing {@link ResourceSiteProperty}.
- * @param value the map from location to the number of Task that must be executed on the node
- * @return the execution property
- */
- public static ResourceSiteProperty of(final HashMap<String, Integer> value) {
- return new ResourceSiteProperty(value);
- }
+ /**
+ * Static method for constructing {@link ResourceSiteProperty}.
+ *
+ * @param value the map from location to the number of Task that must be executed on the node
+ * @return the execution property
+ */
+ public static ResourceSiteProperty of(final HashMap<String, Integer> value) {
+ return new ResourceSiteProperty(value);
+ }
}
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSlotProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSlotProperty.java
index 1b3c161..7475e21 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSlotProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ResourceSlotProperty.java
@@ -26,7 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.VertexExecutionProperty;
public final class ResourceSlotProperty extends VertexExecutionProperty<Boolean> {
private static final ResourceSlotProperty COMPLIANCE_TRUE = new ResourceSlotProperty(true);
private static final ResourceSlotProperty COMPLIANCE_FALSE
- = new ResourceSlotProperty(false);
+ = new ResourceSlotProperty(false);
/**
* Default constructor.
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ScheduleGroupProperty.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ScheduleGroupProperty.java
index f6df810..97e9d6a 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ScheduleGroupProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/executionproperty/ScheduleGroupProperty.java
@@ -26,6 +26,7 @@ import org.apache.nemo.common.ir.executionproperty.VertexExecutionProperty;
public final class ScheduleGroupProperty extends VertexExecutionProperty<Integer> {
/**
* Constructor.
+ *
* @param value value of the execution property.
*/
private ScheduleGroupProperty(final Integer value) {
@@ -34,6 +35,7 @@ public final class ScheduleGroupProperty extends VertexExecutionProperty<Integer
/**
* Static method exposing the constructor.
+ *
* @param value value of the new execution property.
* @return the newly created execution property.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageAggregatorTransform.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageAggregatorTransform.java
index d4dd77b..e33900f 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageAggregatorTransform.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageAggregatorTransform.java
@@ -40,6 +40,7 @@ public final class MessageAggregatorTransform<K, V, O> extends NoWatermarkEmitTr
/**
* Default constructor.
+ *
* @param aggregatedDynOptData per-stage aggregated dynamic optimization data.
* @param dynOptDataAggregator aggregator to use.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageBarrierTransform.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageBarrierTransform.java
index 4c9a007..fe2e523 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageBarrierTransform.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/MessageBarrierTransform.java
@@ -29,6 +29,7 @@ import java.util.function.BiFunction;
/**
* A {@link Transform} that collects task-level statistics used for dynamic optimization.
+ *
* @param <I> input type.
* @param <K> output key type.
* @param <V> output value type.
@@ -42,6 +43,7 @@ public final class MessageBarrierTransform<I, K, V> extends NoWatermarkEmitTrans
/**
* MessageBarrierTransform constructor.
+ *
* @param userFunction that analyzes the data.
*/
public MessageBarrierTransform(final BiFunction<I, Map<K, V>, Map<K, V>> userFunction) {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/NoWatermarkEmitTransform.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/NoWatermarkEmitTransform.java
index b9fab8d..549a3f1 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/NoWatermarkEmitTransform.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/NoWatermarkEmitTransform.java
@@ -23,6 +23,7 @@ import org.apache.nemo.common.punctuation.Watermark;
/**
* This transform does not emit watermarks.
* It may be a transform for batch operation that emits collected data when calling {@link Transform#close()}.
+ *
* @param <I> input type
* @param <O> output type
*/
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/StreamTransform.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/StreamTransform.java
index 97a7013..9781a8d 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/StreamTransform.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/StreamTransform.java
@@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory;
/**
* A {@link Transform} relays input data from upstream vertex to downstream vertex promptly.
* This transform can be used for merging input data into the {@link OutputCollector}.
+ *
* @param <T> input/output type.
*/
public final class StreamTransform<T> implements Transform<T, T> {
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/Transform.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/Transform.java
index a79c192..a4787e1 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/Transform.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/transform/Transform.java
@@ -28,19 +28,22 @@ import java.util.Optional;
* Interface for specifying 'What' to do with data.
* It is to be implemented in the compiler frontend, possibly for every operator in a dataflow language.
* 'How' and 'When' to do with its input/output data are up to the runtime.
+ *
* @param <I> input type.
* @param <O> output type.
*/
public interface Transform<I, O> extends Serializable {
/**
* Prepare the transform.
- * @param context of the transform.
+ *
+ * @param context of the transform.
* @param outputCollector that collects outputs.
*/
void prepare(Context context, OutputCollector<O> outputCollector);
/**
* On data received.
+ *
* @param element data received.
*/
void onData(I element);
@@ -49,6 +52,7 @@ public interface Transform<I, O> extends Serializable {
* On watermark received.
* This method should be called for the minimum watermark among input streams (input watermark).
* Transform may emit collected data after receiving watermarks.
+ *
* @param watermark watermark
*/
void onWatermark(Watermark watermark);
@@ -70,12 +74,14 @@ public interface Transform<I, O> extends Serializable {
/**
* Put serialized data to send to the executor.
+ *
* @param serializedData the serialized data.
*/
void setSerializedData(String serializedData);
/**
* Retrieve the serialized data on the executor.
+ *
* @return the serialized data.
*/
Optional<String> getSerializedData();
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageAggregatorVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageAggregatorVertex.java
index 076ba1b..2e2cdb6 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageAggregatorVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageAggregatorVertex.java
@@ -31,6 +31,7 @@ import java.util.function.BiFunction;
/**
* Aggregates upstream messages.
+ *
* @param <K> of the input pair.
* @param <V> of the input pair.
* @param <O> of the output aggregated message.
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageBarrierVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageBarrierVertex.java
index 66ed760..2647794 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageBarrierVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/MessageBarrierVertex.java
@@ -26,6 +26,7 @@ import java.util.function.BiFunction;
/**
* Generates messages.
+ *
* @param <I> input type
* @param <K> of the output pair.
* @param <V> of the output pair.
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/SamplingVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/SamplingVertex.java
index 65ccfee..b614022 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/SamplingVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/utility/SamplingVertex.java
@@ -32,7 +32,7 @@ public final class SamplingVertex extends IRVertex {
private final float desiredSampleRate;
/**
- * @param originalVertex to clone.
+ * @param originalVertex to clone.
* @param desiredSampleRate percentage of tasks to execute.
* The actual sample rate may vary depending on neighboring sampling vertices.
*/
@@ -80,7 +80,7 @@ public final class SamplingVertex extends IRVertex {
/**
* Obtains a clone of an original edge that is attached to this sampling vertex.
- *
+ * <p>
* Original edge: src - to - dst
* When src == originalVertex, return thisSamplingVertex - to - dst
* When dst == originalVertex, return src - to - thisSamplingVertex
diff --git a/common/src/main/java/org/apache/nemo/common/partitioner/HashPartitioner.java b/common/src/main/java/org/apache/nemo/common/partitioner/HashPartitioner.java
index 655298e..241f056 100644
--- a/common/src/main/java/org/apache/nemo/common/partitioner/HashPartitioner.java
+++ b/common/src/main/java/org/apache/nemo/common/partitioner/HashPartitioner.java
@@ -33,7 +33,7 @@ public final class HashPartitioner implements Partitioner<Integer> {
* Constructor.
*
* @param numOfPartitions the number of partitions.
- * @param keyExtractor the key extractor that extracts keys from elements.
+ * @param keyExtractor the key extractor that extracts keys from elements.
*/
public HashPartitioner(final int numOfPartitions,
final KeyExtractor keyExtractor) {
diff --git a/common/src/main/java/org/apache/nemo/common/partitioner/Partitioner.java b/common/src/main/java/org/apache/nemo/common/partitioner/Partitioner.java
index f389032..dabaf0a 100644
--- a/common/src/main/java/org/apache/nemo/common/partitioner/Partitioner.java
+++ b/common/src/main/java/org/apache/nemo/common/partitioner/Partitioner.java
@@ -48,7 +48,7 @@ public interface Partitioner<K extends Serializable> {
/**
* @param edgeProperties edge properties.
- * @param dstProperties vertex properties.
+ * @param dstProperties vertex properties.
* @return the partitioner.
*/
static Partitioner getPartitioner(final ExecutionPropertyMap<EdgeExecutionProperty> edgeProperties,
diff --git a/common/src/main/java/org/apache/nemo/common/pass/Pass.java b/common/src/main/java/org/apache/nemo/common/pass/Pass.java
index 7a540c9..3000fdb 100644
--- a/common/src/main/java/org/apache/nemo/common/pass/Pass.java
+++ b/common/src/main/java/org/apache/nemo/common/pass/Pass.java
@@ -38,6 +38,7 @@ public abstract class Pass implements Serializable {
/**
* Constructor.
+ *
* @param condition condition under which to run the pass.
*/
private Pass(final Predicate<IRDAG> condition) {
@@ -46,6 +47,7 @@ public abstract class Pass implements Serializable {
/**
* Getter for the condition under which to apply the pass.
+ *
* @return the condition under which to apply the pass.
*/
public final Predicate<IRDAG> getCondition() {
@@ -54,6 +56,7 @@ public abstract class Pass implements Serializable {
/**
* Add the condition to the existing condition to run the pass.
+ *
* @param newCondition the new condition to add to the existing condition.
* @return the condition with the new condition added.
*/
diff --git a/common/src/main/java/org/apache/nemo/common/test/EmptyComponents.java b/common/src/main/java/org/apache/nemo/common/test/EmptyComponents.java
index 45af7cf..fbd53c2 100644
--- a/common/src/main/java/org/apache/nemo/common/test/EmptyComponents.java
+++ b/common/src/main/java/org/apache/nemo/common/test/EmptyComponents.java
@@ -19,6 +19,7 @@
package org.apache.nemo.common.test;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.beam.sdk.values.KV;
import org.apache.nemo.common.KeyExtractor;
import org.apache.nemo.common.coder.DecoderFactory;
import org.apache.nemo.common.coder.EncoderFactory;
@@ -33,7 +34,6 @@ import org.apache.nemo.common.ir.vertex.OperatorVertex;
import org.apache.nemo.common.ir.vertex.SourceVertex;
import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
import org.apache.nemo.common.ir.vertex.transform.Transform;
-import org.apache.beam.sdk.values.KV;
import java.io.IOException;
import java.util.ArrayList;
@@ -65,6 +65,7 @@ public final class EmptyComponents {
/**
* Builds dummy IR DAG for testing.
+ *
* @return the dummy IR DAG.
*/
public static IRDAG buildEmptyDAG() {
@@ -93,6 +94,7 @@ public final class EmptyComponents {
* Builds dummy IR DAG to test skew handling.
* For DataSkewPolicy, shuffle edges needs extra setting for EncoderProperty, DecoderProperty
* and KeyExtractorProperty by default.
+ *
* @return the dummy IR DAG.
*/
public static IRDAG buildEmptyDAGForSkew() {
@@ -198,7 +200,7 @@ public final class EmptyComponents {
/**
* Constructor.
*
- * @param name name for the vertex.
+ * @param name name for the vertex.
* @param minNumReadables for the vertex.
*/
public EmptySourceVertex(final String name, final int minNumReadables) {
diff --git a/common/src/main/java/org/apache/nemo/common/test/ExampleTestUtil.java b/common/src/main/java/org/apache/nemo/common/test/ExampleTestUtil.java
index 42fc486..628a12f 100644
--- a/common/src/main/java/org/apache/nemo/common/test/ExampleTestUtil.java
+++ b/common/src/main/java/org/apache/nemo/common/test/ExampleTestUtil.java
@@ -33,6 +33,7 @@ import java.util.stream.Stream;
*/
public final class ExampleTestUtil {
private static final Double ERROR = 1e-8;
+
/**
* Private constructor.
*/
@@ -42,8 +43,8 @@ public final class ExampleTestUtil {
/**
* Ensures output correctness with the given test resource file.
*
- * @param resourcePath root folder for both resources.
- * @param outputFileName output file name.
+ * @param resourcePath root folder for both resources.
+ * @param outputFileName output file name.
* @param testResourceFileName the test result file name.
* @throws IOException if the output is invalid.
*/
@@ -54,38 +55,38 @@ public final class ExampleTestUtil {
final String testOutput;
try (final Stream<Path> fileStream = Files.list(Paths.get(resourcePath))) {
testOutput = fileStream
- .filter(Files::isRegularFile)
- // TODO 346: Do not use test file prefixes
- // i.e., replace startsWith() with something like regex matching
- .filter(path -> path.getFileName().toString().startsWith(outputFileName))
- .flatMap(path -> {
- try {
- return Files.lines(path);
- } catch (final IOException e) {
- throw new RuntimeException(e);
- }
- })
- .sorted()
- .reduce("", (p, q) -> (p + "\n" + q));
+ .filter(Files::isRegularFile)
+ // TODO 346: Do not use test file prefixes
+ // i.e., replace startsWith() with something like regex matching
+ .filter(path -> path.getFileName().toString().startsWith(outputFileName))
+ .flatMap(path -> {
+ try {
+ return Files.lines(path);
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .sorted()
+ .reduce("", (p, q) -> (p + "\n" + q));
}
final String resourceOutput;
try (final Stream<String> lineStream = Files.lines(Paths.get(resourcePath + testResourceFileName))) {
resourceOutput = lineStream
- .sorted()
- .reduce("", (p, q) -> (p + "\n" + q));
+ .sorted()
+ .reduce("", (p, q) -> (p + "\n" + q));
}
if (!testOutput.equals(resourceOutput)) {
final String outputMsg =
- "Test output mismatch while comparing [" + outputFileName + "] from [" + testResourceFileName + "] under "
- + resourcePath + ":\n"
- + "=============" + outputFileName + "=================="
- + testOutput
- + "\n=============" + testResourceFileName + "=================="
- + resourceOutput
- + "\n===============================";
+ "Test output mismatch while comparing [" + outputFileName + "] from [" + testResourceFileName + "] under "
+ + resourcePath + ":\n"
+ + "=============" + outputFileName + "=================="
+ + testOutput
+ + "\n=============" + testResourceFileName + "=================="
+ + resourceOutput
+ + "\n===============================";
throw new RuntimeException(outputMsg);
}
}
@@ -95,11 +96,11 @@ public final class ExampleTestUtil {
* Due to the floating point math error, the output of the test can be different every time.
* Thus we cannot compare plain text output, but have to check its numeric error.
*
- * @param resourcePath path to resources.
- * @param outputFileName name of output file.
+ * @param resourcePath path to resources.
+ * @param outputFileName name of output file.
* @param testResourceFileName name of the file to compare the outputs to.
* @throws RuntimeException if the output is invalid.
- * @throws IOException exception.
+ * @throws IOException exception.
*/
public static void ensureALSOutputValidity(final String resourcePath,
final String outputFileName,
@@ -108,30 +109,30 @@ public final class ExampleTestUtil {
final List<List<Double>> testOutput;
try (final Stream<Path> fileStream = Files.list(Paths.get(resourcePath))) {
testOutput = fileStream
- .filter(Files::isRegularFile)
- .filter(path -> path.getFileName().toString().startsWith(outputFileName))
- .flatMap(path -> {
- try {
- return Files.lines(path);
- } catch (final IOException e) {
- throw new RuntimeException(e);
- }
- })
- .sorted()
- .filter(line -> !line.trim().equals(""))
- .map(line -> Arrays.asList(line.split("\\s*,\\s*"))
- .stream().map(s -> Double.valueOf(s)).collect(Collectors.toList()))
- .collect(Collectors.toList());
+ .filter(Files::isRegularFile)
+ .filter(path -> path.getFileName().toString().startsWith(outputFileName))
+ .flatMap(path -> {
+ try {
+ return Files.lines(path);
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .sorted()
+ .filter(line -> !line.trim().equals(""))
+ .map(line -> Arrays.asList(line.split("\\s*,\\s*"))
+ .stream().map(s -> Double.valueOf(s)).collect(Collectors.toList()))
+ .collect(Collectors.toList());
}
final List<List<Double>> resourceOutput;
try (final Stream<String> lineStream = Files.lines(Paths.get(resourcePath + testResourceFileName))) {
resourceOutput = lineStream
- .sorted()
- .filter(line -> !line.trim().equals(""))
- .map(line -> Arrays.asList(line.split("\\s*,\\s*"))
- .stream().map(s -> Double.valueOf(s)).collect(Collectors.toList()))
- .collect(Collectors.toList());
+ .sorted()
+ .filter(line -> !line.trim().equals(""))
+ .map(line -> Arrays.asList(line.split("\\s*,\\s*"))
+ .stream().map(s -> Double.valueOf(s)).collect(Collectors.toList()))
+ .collect(Collectors.toList());
}
if (testOutput.size() != resourceOutput.size()) {
@@ -160,9 +161,9 @@ public final class ExampleTestUtil {
final String outputFileName) throws IOException {
try (final Stream<Path> fileStream = Files.list(Paths.get(directory))) {
final Set<Path> outputFilePaths = fileStream
- .filter(Files::isRegularFile)
- .filter(path -> path.getFileName().toString().startsWith(outputFileName))
- .collect(Collectors.toSet());
+ .filter(Files::isRegularFile)
+ .filter(path -> path.getFileName().toString().startsWith(outputFileName))
+ .collect(Collectors.toSet());
for (final Path outputFilePath : outputFilePaths) {
Files.delete(outputFilePath);
}
diff --git a/common/src/test/java/org/apache/nemo/common/DAGTest.java b/common/src/test/java/org/apache/nemo/common/DAGTest.java
index d1841f8..e735fef 100644
--- a/common/src/test/java/org/apache/nemo/common/DAGTest.java
+++ b/common/src/test/java/org/apache/nemo/common/DAGTest.java
@@ -19,8 +19,8 @@
package org.apache.nemo.common;
import org.apache.nemo.common.dag.DAG;
-import org.apache.nemo.common.dag.Edge;
import org.apache.nemo.common.dag.DAGBuilder;
+import org.apache.nemo.common.dag.Edge;
import org.apache.nemo.common.dag.Vertex;
import org.junit.Before;
import org.junit.Test;
@@ -144,6 +144,7 @@ public final class DAGTest {
final class IntegerVertex extends Vertex {
private final int value;
+
public IntegerVertex(final int value) {
super(String.valueOf(value));
this.value = value;
@@ -158,7 +159,7 @@ public final class DAGTest {
if (this == o) {
return true;
}
- if (o == null || getClass() != o.getClass()){
+ if (o == null || getClass() != o.getClass()) {
return false;
}
final IntegerVertex that = (IntegerVertex) o;
diff --git a/common/src/test/java/org/apache/nemo/common/PairTest.java b/common/src/test/java/org/apache/nemo/common/PairTest.java
index 8792131..8b59e12 100644
--- a/common/src/test/java/org/apache/nemo/common/PairTest.java
+++ b/common/src/test/java/org/apache/nemo/common/PairTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.nemo.common;
-import org.apache.nemo.common.Pair;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.test.EmptyComponents;
import org.junit.Test;
diff --git a/common/src/test/java/org/apache/nemo/common/StateMachineTest.java b/common/src/test/java/org/apache/nemo/common/StateMachineTest.java
index 4d1f63f..41f9cd2 100644
--- a/common/src/test/java/org/apache/nemo/common/StateMachineTest.java
+++ b/common/src/test/java/org/apache/nemo/common/StateMachineTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.nemo.common;
-import org.apache.nemo.common.StateMachine;
import org.apache.nemo.common.exception.IllegalStateTransitionException;
import org.junit.Before;
import org.junit.Test;
diff --git a/common/src/test/java/org/apache/nemo/common/coder/CoderFactoryTest.java b/common/src/test/java/org/apache/nemo/common/coder/CoderFactoryTest.java
index 1b3302e..acd4345 100644
--- a/common/src/test/java/org/apache/nemo/common/coder/CoderFactoryTest.java
+++ b/common/src/test/java/org/apache/nemo/common/coder/CoderFactoryTest.java
@@ -55,8 +55,8 @@ public class CoderFactoryTest {
* @return the decoded element.
*/
private <T> T encodeAndDecodeElement(final EncoderFactory<T> encoderFactory,
- final DecoderFactory<T> decoderFactory,
- final T element) throws Exception {
+ final DecoderFactory<T> decoderFactory,
+ final T element) throws Exception {
final byte[] encodedElement;
try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) {
final EncoderFactory.Encoder<T> encoder = encoderFactory.create(out);
diff --git a/common/src/test/java/org/apache/nemo/common/ir/IRDAGTest.java b/common/src/test/java/org/apache/nemo/common/ir/IRDAGTest.java
index 1025c7c..9444c33 100644
--- a/common/src/test/java/org/apache/nemo/common/ir/IRDAGTest.java
+++ b/common/src/test/java/org/apache/nemo/common/ir/IRDAGTest.java
@@ -280,7 +280,8 @@ public class IRDAGTest {
irdag.insert(Sets.newHashSet(svOne), Sets.newHashSet(sourceVertex));
mustPass();
- final SamplingVertex svTwo = new SamplingVertex(firstOperatorVertex, 0.1f);;
+ final SamplingVertex svTwo = new SamplingVertex(firstOperatorVertex, 0.1f);
+ ;
irdag.insert(Sets.newHashSet(svTwo), Sets.newHashSet(firstOperatorVertex));
mustPass();
@@ -319,13 +320,27 @@ public class IRDAGTest {
switch (methodIndex) {
// Annotation methods
// For simplicity, we test only the EPs for which all possible values are valid.
- case 0: selectRandomVertex().setProperty(randomCSP()); break;
- case 1: selectRandomVertex().setProperty(randomRLP()); break;
- case 2: selectRandomVertex().setProperty(randomRPP()); break;
- case 3: selectRandomVertex().setProperty(randomRSP()); break;
- case 4: selectRandomEdge().setProperty(randomDFP()); break;
- case 5: selectRandomEdge().setProperty(randomDPP()); break;
- case 6: selectRandomEdge().setProperty(randomDSP()); break;
+ case 0:
+ selectRandomVertex().setProperty(randomCSP());
+ break;
+ case 1:
+ selectRandomVertex().setProperty(randomRLP());
+ break;
+ case 2:
+ selectRandomVertex().setProperty(randomRPP());
+ break;
+ case 3:
+ selectRandomVertex().setProperty(randomRSP());
+ break;
+ case 4:
+ selectRandomEdge().setProperty(randomDFP());
+ break;
+ case 5:
+ selectRandomEdge().setProperty(randomDPP());
+ break;
+ case 6:
+ selectRandomEdge().setProperty(randomDSP());
+ break;
// Reshaping methods
case 7:
@@ -434,11 +449,16 @@ public class IRDAGTest {
private DataStoreProperty randomDSP() {
switch (random.nextInt(4)) {
- case 0: return DataStoreProperty.of(DataStoreProperty.Value.MemoryStore);
- case 1: return DataStoreProperty.of(DataStoreProperty.Value.SerializedMemoryStore);
- case 2: return DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore);
- case 3: return DataStoreProperty.of(DataStoreProperty.Value.GlusterFileStore);
- default: throw new IllegalStateException();
+ case 0:
+ return DataStoreProperty.of(DataStoreProperty.Value.MemoryStore);
+ case 1:
+ return DataStoreProperty.of(DataStoreProperty.Value.SerializedMemoryStore);
+ case 2:
+ return DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore);
+ case 3:
+ return DataStoreProperty.of(DataStoreProperty.Value.GlusterFileStore);
+ default:
+ throw new IllegalStateException();
}
}
}
diff --git a/common/src/test/java/org/apache/nemo/common/ir/LoopVertexTest.java b/common/src/test/java/org/apache/nemo/common/ir/LoopVertexTest.java
index da500b1..9bf339b 100644
--- a/common/src/test/java/org/apache/nemo/common/ir/LoopVertexTest.java
+++ b/common/src/test/java/org/apache/nemo/common/ir/LoopVertexTest.java
@@ -18,13 +18,13 @@
*/
package org.apache.nemo.common.ir;
+import org.apache.nemo.common.Pair;
+import org.apache.nemo.common.dag.DAGBuilder;
import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.ir.vertex.LoopVertex;
import org.apache.nemo.common.ir.vertex.OperatorVertex;
-import org.apache.nemo.common.Pair;
-import org.apache.nemo.common.dag.DAGBuilder;
import org.apache.nemo.common.test.EmptyComponents;
import org.junit.Before;
import org.junit.Test;
@@ -55,18 +55,18 @@ public class LoopVertexTest {
final DAGBuilder<IRVertex, IREdge> builder = new DAGBuilder<>();
loopDAGBuilder.addVertex(map1).addVertex(groupByKey).addVertex(combine).addVertex(map2)
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.Shuffle, map1, groupByKey))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, groupByKey, combine))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, combine, map2));
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.Shuffle, map1, groupByKey))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, groupByKey, combine))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, combine, map2));
loopVertex.addDagIncomingEdge(new IREdge(CommunicationPatternProperty.Value.OneToOne, source, map1));
loopVertex.addIterativeIncomingEdge(new IREdge(CommunicationPatternProperty.Value.OneToOne, map2, map1));
originalDAG = new IRDAG(builder.addVertex(source).addVertex(map1).addVertex(groupByKey).addVertex(combine).addVertex(map2)
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, source, map1))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.Shuffle, map1, groupByKey))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, groupByKey, combine))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, combine, map2))
- .build());
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, source, map1))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.Shuffle, map1, groupByKey))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, groupByKey, combine))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, combine, map2))
+ .build());
}
@Test
@@ -77,13 +77,13 @@ public class LoopVertexTest {
vertices1.next();
final Iterator<IRVertex> vertices2 = loopVertex.getDAG().getTopologicalSort().iterator();
final List<Pair<IRVertex, IRVertex>> list = new ArrayList<>();
- while (vertices1.hasNext() && vertices2.hasNext()) {
+ while (vertices1.hasNext() && vertices2.hasNext()) {
list.add(Pair.of(vertices1.next(), vertices2.next()));
}
list.forEach(irVertexPair -> {
- assertEquals(irVertexPair.left().getExecutionProperties(), irVertexPair.right().getExecutionProperties());
- assertEquals(originalDAG.getOutgoingEdgesOf(irVertexPair.left()).size(),
- loopVertex.getDAG().getOutgoingEdgesOf(irVertexPair.right()).size());
+ assertEquals(irVertexPair.left().getExecutionProperties(), irVertexPair.right().getExecutionProperties());
+ assertEquals(originalDAG.getOutgoingEdgesOf(irVertexPair.left()).size(),
+ loopVertex.getDAG().getOutgoingEdgesOf(irVertexPair.right()).size());
});
assertEquals(source, loopVertex.getDagIncomingEdges().values().iterator().next().iterator().next().getSrc());
diff --git a/common/src/test/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMapTest.java b/common/src/test/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMapTest.java
index 69af0d2..06e6d58 100644
--- a/common/src/test/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMapTest.java
+++ b/common/src/test/java/org/apache/nemo/common/ir/executionproperty/ExecutionPropertyMapTest.java
@@ -32,9 +32,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
/**
* Test {@link ExecutionPropertyMap}.
diff --git a/common/src/test/java/org/apache/nemo/common/util/UtilTest.java b/common/src/test/java/org/apache/nemo/common/util/UtilTest.java
index e46db33..4e3d02d 100644
--- a/common/src/test/java/org/apache/nemo/common/util/UtilTest.java
+++ b/common/src/test/java/org/apache/nemo/common/util/UtilTest.java
@@ -18,25 +18,25 @@
*/
package org.apache.nemo.common.util;
-import static org.junit.Assert.assertEquals;
+import org.apache.nemo.common.Util;
+import org.junit.Test;
import java.util.function.IntPredicate;
-import org.apache.nemo.common.Util;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
public class UtilTest {
- @Test
- public void testCheckEqualityOfIntPredicates() {
+ @Test
+ public void testCheckEqualityOfIntPredicates() {
- IntPredicate firstPredicate = number -> number < 5;
- IntPredicate secondPredicate = number -> number < 10;
- assertEquals(true,
- Util.checkEqualityOfIntPredicates(firstPredicate, secondPredicate, 4));
- assertEquals(false,
- Util.checkEqualityOfIntPredicates(firstPredicate, secondPredicate, 5));
- assertEquals(false,
- Util.checkEqualityOfIntPredicates(firstPredicate, secondPredicate, 7));
- }
+ IntPredicate firstPredicate = number -> number < 5;
+ IntPredicate secondPredicate = number -> number < 10;
+ assertEquals(true,
+ Util.checkEqualityOfIntPredicates(firstPredicate, secondPredicate, 4));
+ assertEquals(false,
+ Util.checkEqualityOfIntPredicates(firstPredicate, secondPredicate, 5));
+ assertEquals(false,
+ Util.checkEqualityOfIntPredicates(firstPredicate, secondPredicate, 7));
+ }
}
diff --git a/compiler/backend/pom.xml b/compiler/backend/pom.xml
index 237ce32..625a313 100644
--- a/compiler/backend/pom.xml
+++ b/compiler/backend/pom.xml
@@ -17,34 +17,35 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler</artifactId>
- <version>0.2-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
- <artifactId>nemo-compiler-backend</artifactId>
- <name>Nemo Compiler Backend</name>
+ <artifactId>nemo-compiler-backend</artifactId>
+ <name>Nemo Compiler Backend</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-runtime-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler-optimizer</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-runtime-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler-optimizer</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
</project>
diff --git a/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/Backend.java b/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/Backend.java
index 2a4fc20..73f1eb9 100644
--- a/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/Backend.java
+++ b/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/Backend.java
@@ -24,6 +24,7 @@ import org.apache.reef.tang.annotations.DefaultImplementation;
/**
* Interface for backend components.
+ *
* @param <Plan> the physical execution plan to compile the DAG into.
*/
@DefaultImplementation(NemoBackend.class)
diff --git a/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoBackend.java b/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoBackend.java
index 8d6e6d9..c9921c7 100644
--- a/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoBackend.java
+++ b/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoBackend.java
@@ -38,6 +38,7 @@ public final class NemoBackend implements Backend<PhysicalPlan> {
/**
* Constructor.
+ *
* @param physicalPlanGenerator physical plan generator.
*/
@Inject
diff --git a/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoPlanRewriter.java b/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoPlanRewriter.java
index 03373d5..550699a 100644
--- a/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoPlanRewriter.java
+++ b/compiler/backend/src/main/java/org/apache/nemo/compiler/backend/nemo/NemoPlanRewriter.java
@@ -34,19 +34,22 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
/**
* Rewrites the physical plan during execution, to enforce the optimizations of Nemo RunTimePasses.
- *
+ * <p>
* A high-level flow of a rewrite is as follows:
* Runtime - (PhysicalPlan-level info) - NemoPlanRewriter - (IRDAG-level info) - NemoOptimizer - (new IRDAG)
* - NemoPlanRewriter - (new PhysicalPlan) - Runtime
- *
+ * <p>
* Here, the NemoPlanRewriter acts as a translator between the Runtime that only understands PhysicalPlan-level info,
* and the NemoOptimizer that only understands IRDAG-level info.
- *
+ * <p>
* This decoupling between the NemoOptimizer and the Runtime lets Nemo optimization policies dynamically control
* distributed execution behaviors, and at the same time enjoy correctness/reusability/composability properties that
* the IRDAG abstraction provides.
diff --git a/compiler/backend/src/test/java/org/apache/nemo/compiler/backend/nemo/NemoBackendTest.java b/compiler/backend/src/test/java/org/apache/nemo/compiler/backend/nemo/NemoBackendTest.java
index ae851fa..12797d2 100644
--- a/compiler/backend/src/test/java/org/apache/nemo/compiler/backend/nemo/NemoBackendTest.java
+++ b/compiler/backend/src/test/java/org/apache/nemo/compiler/backend/nemo/NemoBackendTest.java
@@ -18,12 +18,12 @@
*/
package org.apache.nemo.compiler.backend.nemo;
+import org.apache.nemo.common.dag.DAGBuilder;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.ir.vertex.OperatorVertex;
-import org.apache.nemo.common.dag.DAGBuilder;
import org.apache.nemo.common.test.EmptyComponents;
import org.apache.nemo.compiler.optimizer.policy.TransientResourcePolicy;
import org.apache.nemo.conf.JobConf;
@@ -53,11 +53,11 @@ public final class NemoBackendTest<I, O> {
@Before
public void setUp() throws Exception {
this.dag = new IRDAG(builder.addVertex(source).addVertex(map1).addVertex(groupByKey).addVertex(combine).addVertex(map2)
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, source, map1))
- .connectVertices(EmptyComponents.newDummyShuffleEdge(map1, groupByKey))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, groupByKey, combine))
- .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, combine, map2))
- .build());
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, source, map1))
+ .connectVertices(EmptyComponents.newDummyShuffleEdge(map1, groupByKey))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, groupByKey, combine))
+ .connectVertices(new IREdge(CommunicationPatternProperty.Value.OneToOne, combine, map2))
+ .build());
this.dag = new TransientResourcePolicy().runCompileTimeOptimization(dag, EMPTY_DAG_DIRECTORY);
diff --git a/compiler/frontend/beam/pom.xml b/compiler/frontend/beam/pom.xml
index c999fff..0c19975 100644
--- a/compiler/frontend/beam/pom.xml
+++ b/compiler/frontend/beam/pom.xml
@@ -15,7 +15,8 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/BeamKeyExtractor.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/BeamKeyExtractor.java
index 9ea0591..61818c5 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/BeamKeyExtractor.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/BeamKeyExtractor.java
@@ -19,9 +19,9 @@
package org.apache.nemo.compiler.frontend.beam;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.nemo.common.KeyExtractor;
-import org.apache.beam.sdk.values.KV;
/**
* Extracts the key from a KV element.
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/InMemorySideInputReader.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/InMemorySideInputReader.java
index f1bb460..c8fe064 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/InMemorySideInputReader.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/InMemorySideInputReader.java
@@ -29,7 +29,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
/**
* Accumulates and provides side inputs in memory.
@@ -80,7 +83,8 @@ public final class InMemorySideInputReader implements ReadyCheckingSideInputRead
/**
* Stores the side input in memory to be used with main inputs.
- * @param view of the side input.
+ *
+ * @param view of the side input.
* @param sideInputElement to add.
*/
public void addSideInputElement(final PCollectionView<?> view,
@@ -94,6 +98,7 @@ public final class InMemorySideInputReader implements ReadyCheckingSideInputRead
* Say a DoFn of this reader has 3 main inputs and 4 side inputs.
* Nemo runtime guarantees that the watermark here
* is the minimum of the all 7 input streams.
+ *
* @param newWatermark to set.
*/
public void setCurrentWatermarkOfAllMainAndSideInputs(final long newWatermark) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslationContext.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslationContext.java
index aec816a..6a4c046 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslationContext.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslationContext.java
@@ -39,7 +39,9 @@ import org.apache.nemo.compiler.frontend.beam.coder.BeamEncoderFactory;
import org.apache.nemo.compiler.frontend.beam.coder.SideInputCoder;
import org.apache.nemo.compiler.frontend.beam.transform.*;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
/**
* A collection of translators for the Beam PTransforms.
@@ -55,7 +57,7 @@ final class PipelineTranslationContext {
private final Pipeline pipeline;
/**
- * @param pipeline the pipeline to translate
+ * @param pipeline the pipeline to translate
* @param pipelineOptions {@link PipelineOptions}
*/
PipelineTranslationContext(final Pipeline pipeline,
@@ -101,13 +103,13 @@ final class PipelineTranslationContext {
/**
* Say the dstIRVertex consumes three views: view0, view1, and view2.
- *
+ * <p>
* We translate that as the following:
* view0 -> SideInputTransform(index=0) ->
* view1 -> SideInputTransform(index=1) -> dstIRVertex(with a map from indices to PCollectionViews)
* view2 -> SideInputTransform(index=2) ->
*
- * @param dstVertex vertex.
+ * @param dstVertex vertex.
* @param sideInputs of the vertex.
*/
void addSideInputEdges(final IRVertex dstVertex, final Map<Integer, PCollectionView<?>> sideInputs) {
@@ -145,7 +147,7 @@ final class PipelineTranslationContext {
/**
* Add IR edge to the builder.
*
- * @param dst the destination IR vertex.
+ * @param dst the destination IR vertex.
* @param input the {@link PValue} {@code dst} consumes
*/
void addEdgeTo(final IRVertex dst, final PValue input) {
@@ -171,9 +173,9 @@ final class PipelineTranslationContext {
}
/**
- * @param edge IR edge to add.
+ * @param edge IR edge to add.
* @param elementCoder element coder.
- * @param windowCoder window coder.
+ * @param windowCoder window coder.
*/
void addEdge(final IREdge edge, final Coder elementCoder, final Coder windowCoder) {
edge.setProperty(KeyExtractorProperty.of(new BeamKeyExtractor()));
@@ -192,9 +194,10 @@ final class PipelineTranslationContext {
/**
* Registers a {@link PValue} as a m.forEach(outputFromGbk -> ain output from the specified {@link IRVertex}.
- * @param node node
+ *
+ * @param node node
* @param irVertex the IR vertex
- * @param output the {@link PValue} {@code irVertex} emits as main output
+ * @param output the {@link PValue} {@code irVertex} emits as main output
*/
void registerMainOutputFrom(final TransformHierarchy.Node node,
final IRVertex irVertex,
@@ -206,10 +209,10 @@ final class PipelineTranslationContext {
/**
* Registers a {@link PValue} as an additional output from the specified {@link IRVertex}.
*
- * @param node node
+ * @param node node
* @param irVertex the IR vertex
- * @param output the {@link PValue} {@code irVertex} emits as additional output
- * @param tag the {@link TupleTag} associated with this additional output
+ * @param output the {@link PValue} {@code irVertex} emits as additional output
+ * @param tag the {@link TupleTag} associated with this additional output
*/
void registerAdditionalOutputFrom(final TransformHierarchy.Node node,
final IRVertex irVertex,
@@ -284,7 +287,8 @@ final class PipelineTranslationContext {
/**
* Get appropriate coder for {@link PCollectionView}.
- * @param view {@link PCollectionView}
+ *
+ * @param view {@link PCollectionView}
* @param context translation context.
* @return appropriate {@link Coder} for {@link PCollectionView}
*/
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
index 816c539..1f91e57 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
@@ -23,11 +23,22 @@ import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.construction.ParDoTranslation;
import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.*;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
import org.apache.nemo.common.ir.vertex.IRVertex;
@@ -36,20 +47,19 @@ import org.apache.nemo.common.ir.vertex.transform.Transform;
import org.apache.nemo.compiler.frontend.beam.source.BeamBoundedSourceVertex;
import org.apache.nemo.compiler.frontend.beam.source.BeamUnboundedSourceVertex;
import org.apache.nemo.compiler.frontend.beam.transform.*;
-import org.apache.beam.sdk.coders.*;
-import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.transforms.*;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.values.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.lang.annotation.*;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -93,7 +103,7 @@ final class PipelineTranslator {
}
/**
- * @param context provides translation context.
+ * @param context provides translation context.
* @param primitive primitive node.
*/
void translatePrimitive(final PipelineTranslationContext context,
@@ -118,7 +128,7 @@ final class PipelineTranslator {
}
/**
- * @param context context.
+ * @param context context.
* @param composite transform.
* @return behavior controls whether or not child transforms are visited.
*/
@@ -175,8 +185,8 @@ final class PipelineTranslator {
/////////////////////// PRIMITIVE TRANSFORMS
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(Read.Unbounded.class)
@@ -190,8 +200,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(Read.Bounded.class)
@@ -205,8 +215,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(ParDo.SingleOutput.class)
@@ -226,8 +236,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(ParDo.MultiOutput.class)
@@ -252,8 +262,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(GroupByKey.class)
@@ -267,8 +277,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator({Window.class, Window.Assign.class})
@@ -291,8 +301,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(View.CreatePCollectionView.class)
@@ -307,8 +317,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
*/
@PrimitiveTransformTranslator(Flatten.PCollections.class)
@@ -329,8 +339,8 @@ final class PipelineTranslator {
* ({@link Combine.Globally} internally uses {@link Combine.PerKey} which will also be optimized by this translator)
* Here, we translate this composite transform as a whole, exploiting its accumulator semantics.
*
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
* @return behavior controls whether or not child transforms are visited.
*/
@@ -388,8 +398,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context
- * @param beamNode the beam node to be translated
+ * @param ctx provides translation context
+ * @param beamNode the beam node to be translated
* @param transform transform which can be obtained from {@code beamNode}
* @return behavior controls whether or not child transforms are visited.
*/
@@ -415,8 +425,8 @@ final class PipelineTranslator {
}
/**
- * @param ctx provides translation context.
- * @param beamNode the beam node to be translated.
+ * @param ctx provides translation context.
+ * @param beamNode the beam node to be translated.
* @param sideInputMap side inputs.
* @return the created DoFnTransform.
*/
@@ -480,7 +490,8 @@ final class PipelineTranslator {
/**
* Create a group by key transform.
* It returns GroupByKeyAndWindowDoFnTransform if window function is not default.
- * @param ctx translation context
+ *
+ * @param ctx translation context
* @param beamNode the beam node to be translated
* @return group by key transform
*/
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineVisitor.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineVisitor.java
index fa3d00d..4cc6574 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineVisitor.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineVisitor.java
@@ -32,7 +32,7 @@ public final class PipelineVisitor extends Pipeline.PipelineVisitor.Defaults {
private final PipelineTranslationContext context;
/**
- * @param pipeline to visit.
+ * @param pipeline to visit.
* @param pipelineOptions pipeline options.
*/
public PipelineVisitor(final Pipeline pipeline, final NemoPipelineOptions pipelineOptions) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/SideInputElement.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/SideInputElement.java
index 7ec91cc..75d10fa 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/SideInputElement.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/SideInputElement.java
@@ -21,6 +21,7 @@ package org.apache.nemo.compiler.frontend.beam;
/**
* {@link org.apache.nemo.compiler.frontend.beam.transform.DoFnTransform} treats elements of this type as side inputs.
* TODO #289: Prevent using SideInputElement in UDFs
+ *
* @param <T> type of the side input value.
*/
public final class SideInputElement<T> {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamDecoderFactory.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamDecoderFactory.java
index 12a20d3..9302bf7 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamDecoderFactory.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamDecoderFactory.java
@@ -18,10 +18,10 @@
*/
package org.apache.nemo.compiler.frontend.beam.coder;
-import org.apache.nemo.common.coder.DecoderFactory;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.nemo.common.coder.DecoderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +30,7 @@ import java.io.InputStream;
/**
* {@link DecoderFactory} from {@link org.apache.beam.sdk.coders.Coder}.
+ *
* @param <T> the type of element to decode.
*/
public final class BeamDecoderFactory<T> implements DecoderFactory<T> {
@@ -62,6 +63,7 @@ public final class BeamDecoderFactory<T> implements DecoderFactory<T> {
/**
* Abstract class for Beam Decoder.
+ *
* @param <T2> the type of element to decode.
*/
private abstract class BeamAbstractDecoder<T2> implements Decoder<T2> {
@@ -105,6 +107,7 @@ public final class BeamDecoderFactory<T> implements DecoderFactory<T> {
/**
* Beam Decoder for non void objects.
+ *
* @param <T2> the type of element to decode.
*/
private final class BeamDecoder<T2> extends BeamAbstractDecoder<T2> {
@@ -133,6 +136,7 @@ public final class BeamDecoderFactory<T> implements DecoderFactory<T> {
/**
* Beam Decoder for {@link VoidCoder}.
+ *
* @param <T2> the type of element to decode.
*/
private final class BeamVoidDecoder<T2> extends BeamAbstractDecoder<T2> {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamEncoderFactory.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamEncoderFactory.java
index a000ead..a60cae8 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamEncoderFactory.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/BeamEncoderFactory.java
@@ -18,10 +18,10 @@
*/
package org.apache.nemo.compiler.frontend.beam.coder;
-import org.apache.nemo.common.coder.EncoderFactory;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.nemo.common.coder.EncoderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +30,7 @@ import java.io.OutputStream;
/**
* {@link EncoderFactory} from {@link Coder}.
+ *
* @param <T> the type of element to encode.
*/
public final class BeamEncoderFactory<T> implements EncoderFactory<T> {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/SideInputCoder.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/SideInputCoder.java
index 8cefa4b..9660b36 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/SideInputCoder.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/coder/SideInputCoder.java
@@ -28,6 +28,7 @@ import java.util.List;
/**
* EncoderFactory for side inputs.
+ *
* @param <T> type of the side input value.
*/
public final class SideInputCoder<T> extends StructuredCoder<SideInputElement<T>> {
@@ -35,6 +36,7 @@ public final class SideInputCoder<T> extends StructuredCoder<SideInputElement<T>
/**
* Private constructor.
+ *
* @param valueCoder the coder for side input.
*/
private SideInputCoder(final Coder<T> valueCoder) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamBoundedSourceVertex.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamBoundedSourceVertex.java
index 07f3a0e..826e00c 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamBoundedSourceVertex.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamBoundedSourceVertex.java
@@ -39,6 +39,7 @@ import java.util.List;
/**
* SourceVertex implementation for BoundedSource.
+ *
* @param <O> output type.
*/
public final class BeamBoundedSourceVertex<O> extends SourceVertex<WindowedValue<O>> {
@@ -50,7 +51,7 @@ public final class BeamBoundedSourceVertex<O> extends SourceVertex<WindowedValue
/**
* Constructor of BeamBoundedSourceVertex.
*
- * @param source BoundedSource to read from.
+ * @param source BoundedSource to read from.
* @param displayData data to display.
*/
public BeamBoundedSourceVertex(final BoundedSource<O> source, final DisplayData displayData) {
@@ -125,6 +126,7 @@ public final class BeamBoundedSourceVertex<O> extends SourceVertex<WindowedValue
/**
* BoundedSourceReadable class.
+ *
* @param <T> type.
*/
private static final class BoundedSourceReadable<T> implements Readable<WindowedValue<T>> {
@@ -134,6 +136,7 @@ public final class BeamBoundedSourceVertex<O> extends SourceVertex<WindowedValue
/**
* Constructor of the BoundedSourceReadable.
+ *
* @param boundedSource the BoundedSource.
*/
BoundedSourceReadable(final BoundedSource<T> boundedSource) {
@@ -184,7 +187,7 @@ public final class BeamBoundedSourceVertex<O> extends SourceVertex<WindowedValue
final Field inputSplitField = boundedSource.getClass().getDeclaredField("inputSplit");
inputSplitField.setAccessible(true);
final InputSplit inputSplit = ((HadoopFormatIO.SerializableSplit) inputSplitField
- .get(boundedSource)).getSplit();
+ .get(boundedSource)).getSplit();
return Arrays.asList(inputSplit.getLocations());
} else {
throw new UnsupportedOperationException();
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamUnboundedSourceVertex.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamUnboundedSourceVertex.java
index 7371842..44706bd 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamUnboundedSourceVertex.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/source/BeamUnboundedSourceVertex.java
@@ -38,6 +38,7 @@ import java.util.NoSuchElementException;
/**
* SourceVertex implementation for UnboundedSource.
+ *
* @param <O> output type.
* @param <M> checkpoint mark type.
*/
@@ -50,7 +51,8 @@ public final class BeamUnboundedSourceVertex<O, M extends UnboundedSource.Checkp
/**
* The default constructor for beam unbounded source.
- * @param source unbounded source.
+ *
+ * @param source unbounded source.
* @param displayData static display data associated with a pipeline component.
*/
public BeamUnboundedSourceVertex(final UnboundedSource<O, M> source,
@@ -61,6 +63,7 @@ public final class BeamUnboundedSourceVertex<O, M extends UnboundedSource.Checkp
/**
* Copy constructor.
+ *
* @param that the original vertex.
*/
private BeamUnboundedSourceVertex(final BeamUnboundedSourceVertex<O, M> that) {
@@ -113,11 +116,12 @@ public final class BeamUnboundedSourceVertex<O, M extends UnboundedSource.Checkp
/**
* UnboundedSourceReadable class.
+ *
* @param <O> output type.
* @param <M> checkpoint mark type.
*/
private static final class UnboundedSourceReadable<O, M extends UnboundedSource.CheckpointMark>
- implements Readable<Object> {
+ implements Readable<Object> {
private final UnboundedSource<O, M> unboundedSource;
private UnboundedSource.UnboundedReader<O> reader;
private boolean isStarted = false;
@@ -126,6 +130,7 @@ public final class BeamUnboundedSourceVertex<O, M extends UnboundedSource.Checkp
/**
* Constructor.
+ *
* @param unboundedSource unbounded source.
*/
UnboundedSourceReadable(final UnboundedSource<O, M> unboundedSource) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
index a8a44a8..1139282 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
@@ -44,8 +44,8 @@ import java.util.Map;
/**
* This is a base class for Beam DoFn Transforms.
*
- * @param <InputT> input type.
- * @param <InterT> intermediate type.
+ * @param <InputT> input type.
+ * @param <InterT> intermediate type.
* @param <OutputT> output type.
*/
public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
@@ -84,15 +84,16 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* AbstractDoFnTransform constructor.
- * @param doFn doFn
- * @param inputCoder input coder
- * @param outputCoders output coders
- * @param mainOutputTag main output tag
+ *
+ * @param doFn doFn
+ * @param inputCoder input coder
+ * @param outputCoders output coders
+ * @param mainOutputTag main output tag
* @param additionalOutputTags additional output tags
- * @param windowingStrategy windowing strategy
- * @param sideInputs side inputs
- * @param options pipeline options
- * @param displayData display data.
+ * @param windowingStrategy windowing strategy
+ * @param sideInputs side inputs
+ * @param options pipeline options
+ * @param displayData display data.
*/
public AbstractDoFnTransform(final DoFn<InterT, OutputT> doFn,
final Coder<InputT> inputCoder,
@@ -116,6 +117,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for side inputs.
+ *
* @return the side inputs
*/
final Map<Integer, PCollectionView<?>> getSideInputs() {
@@ -124,6 +126,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for output manager.
+ *
* @return the output manager.
*/
final DoFnRunners.OutputManager getOutputManager() {
@@ -132,6 +135,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for windowing strategy.
+ *
* @return the windowing strategy.
*/
final WindowingStrategy getWindowingStrategy() {
@@ -140,6 +144,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for output tag.
+ *
* @return main output tag.
*/
final TupleTag<OutputT> getMainOutputTag() {
@@ -148,6 +153,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for DoFn runner.
+ *
* @return DoFn runner.
*/
final DoFnRunner<InterT, OutputT> getDoFnRunner() {
@@ -156,6 +162,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for push back runner.
+ *
* @return push back runner.
*/
final PushbackSideInputDoFnRunner<InterT, OutputT> getPushBackRunner() {
@@ -164,6 +171,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for side input reader.
+ *
* @return side input reader.
*/
final InMemorySideInputReader getSideInputReader() {
@@ -172,6 +180,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for DoFn.
+ *
* @return DoFn.
*/
public final DoFn getDoFn() {
@@ -181,7 +190,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Checks whether the bundle is finished or not.
* Starts the bundle if it is done.
- *
+ * <p>
* TODO #263: Partial Combining for Beam Streaming
* We may want to use separate methods for doFnRunner/pushBackRunner
* (same applies to the other bundle-related methods)
@@ -251,6 +260,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
public StateInternals stateInternals() {
throw new UnsupportedOperationException("Not support stateInternals in DoFnTransform");
}
+
@Override
public TimerInternals timerInternals() {
throw new UnsupportedOperationException("Not support timerInternals in DoFnTransform");
@@ -284,6 +294,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* Getter function for output collector.
+ *
* @return output collector.
*/
public final OutputCollector<WindowedValue<OutputT>> getOutputCollector() {
@@ -304,6 +315,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* An abstract function that wraps the original doFn.
+ *
* @param originalDoFn the original doFn.
* @return wrapped doFn.
*/
@@ -311,6 +323,7 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
/**
* An abstract function that wraps the original output collector.
+ *
* @param oc the original outputCollector.
* @return wrapped output collector.
*/
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnFinalTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnFinalTransform.java
index 9ef17a2..8f7d2e2 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnFinalTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnFinalTransform.java
@@ -38,6 +38,7 @@ import java.util.Map;
* (Currently supports batch-style global windows only)
* TODO #263: Partial Combining for Beam Streaming
* TODO #264: Partial Combining with Beam SideInputs
+ *
* @param <K> Key type.
* @param <A> Accum type.
* @param <O> Output type.
@@ -53,6 +54,7 @@ public final class CombineFnFinalTransform<K, A, O>
/**
* Constructor.
+ *
* @param combineFn combine function.
*/
public CombineFnFinalTransform(final CombineFnBase.GlobalCombineFn<?, A, O> combineFn) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnPartialTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnPartialTransform.java
index 5f46f90..c620eb7 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnPartialTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CombineFnPartialTransform.java
@@ -37,6 +37,7 @@ import java.util.Map;
* (Currently supports batch-style global windows only)
* TODO #263: Partial Combining for Beam Streaming
* TODO #264: Partial Combining with Beam SideInputs
+ *
* @param <K> Key type.
* @param <I> Input type.
* @param <A> Accum type.
@@ -52,6 +53,7 @@ public final class CombineFnPartialTransform<K, I, A>
/**
* Constructor.
+ *
* @param combineFn combine function.
*/
public CombineFnPartialTransform(final CombineFnBase.GlobalCombineFn<I, A, ?> combineFn) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java
index 8444f0b..bf0e274 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java
@@ -18,13 +18,13 @@
*/
package org.apache.nemo.compiler.frontend.beam.transform;
+import org.apache.beam.sdk.transforms.Materializations;
+import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.nemo.common.ir.OutputCollector;
-import org.apache.beam.sdk.transforms.Materializations;
-import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.values.KV;
+import org.apache.nemo.common.ir.OutputCollector;
import org.apache.nemo.common.ir.vertex.transform.Transform;
import org.apache.nemo.common.punctuation.Watermark;
@@ -34,6 +34,7 @@ import java.util.*;
/**
* This transforms emits materialized data for each window.
+ *
* @param <I> input type
* @param <O> materialized output type
*/
@@ -47,9 +48,10 @@ public final class CreateViewTransform<I, O> implements Transform<WindowedValue<
/**
* Constructor of CreateViewTransform.
+ *
* @param viewFn the viewFn that materializes data.
*/
- public CreateViewTransform(final ViewFn<Materializations.MultimapView<Void, ?>, O> viewFn) {
+ public CreateViewTransform(final ViewFn<Materializations.MultimapView<Void, ?>, O> viewFn) {
this.viewFn = viewFn;
this.windowListMap = new HashMap<>();
this.currentOutputWatermark = Long.MIN_VALUE;
@@ -119,6 +121,7 @@ public final class CreateViewTransform<I, O> implements Transform<WindowedValue<
/**
* Represents {@code PrimitiveViewT} supplied to the {@link ViewFn}.
+ *
* @param <T> primitive view type
*/
public static final class MultiView<T> implements Materializations.MultimapView<Void, T>, Serializable {
@@ -126,6 +129,7 @@ public final class CreateViewTransform<I, O> implements Transform<WindowedValue<
/**
* Constructor.
+ *
* @param iterable placeholder for side input data.
*/
public MultiView(final Iterable<T> iterable) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DefaultOutputManager.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DefaultOutputManager.java
index 16b4e54..bcb5f88 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DefaultOutputManager.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DefaultOutputManager.java
@@ -25,6 +25,7 @@ import org.apache.nemo.common.ir.OutputCollector;
/**
* Default output emitter that uses outputCollector.
+ *
* @param <OutputT> output type
*/
public final class DefaultOutputManager<OutputT> implements DoFnRunners.OutputManager {
@@ -33,7 +34,7 @@ public final class DefaultOutputManager<OutputT> implements DoFnRunners.OutputMa
/**
* @param outputCollector output collector.
- * @param mainOutputTag main output tag.
+ * @param mainOutputTag main output tag.
*/
DefaultOutputManager(final OutputCollector<WindowedValue<OutputT>> outputCollector,
final TupleTag<OutputT> mainOutputTag) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
index 1afc69b..2748b02 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
@@ -37,7 +37,7 @@ import java.util.Map;
/**
* DoFn transform implementation when there is no side input.
*
- * @param <InputT> input type.
+ * @param <InputT> input type.
* @param <OutputT> output type.
*/
public final class DoFnTransform<InputT, OutputT> extends AbstractDoFnTransform<InputT, InputT, OutputT> {
@@ -45,14 +45,15 @@ public final class DoFnTransform<InputT, OutputT> extends AbstractDoFnTransform<
/**
* DoFnTransform Constructor.
- * @param doFn doFn
- * @param inputCoder input coder
- * @param outputCoders output coders
- * @param mainOutputTag main output tag
+ *
+ * @param doFn doFn
+ * @param inputCoder input coder
+ * @param outputCoders output coders
+ * @param mainOutputTag main output tag
* @param additionalOutputTags additional output tags
- * @param windowingStrategy windowing strategy
- * @param options pipeline options
- * @param displayData display data.
+ * @param windowingStrategy windowing strategy
+ * @param options pipeline options
+ * @param displayData display data.
*/
public DoFnTransform(final DoFn<InputT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/FlattenTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/FlattenTransform.java
index 082e5d7..af1146f 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/FlattenTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/FlattenTransform.java
@@ -24,6 +24,7 @@ import org.apache.nemo.common.punctuation.Watermark;
/**
* Flatten transform implementation.
+ *
* @param <T> input/output type.
*/
public final class FlattenTransform<T> implements Transform<T, T> {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java
index f3a2430..cf75cb4 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java
@@ -26,9 +26,9 @@ import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
-import org.apache.beam.sdk.values.KV;
import org.apache.nemo.common.ir.OutputCollector;
import org.apache.nemo.common.punctuation.Watermark;
import org.joda.time.Instant;
@@ -39,7 +39,8 @@ import java.util.*;
/**
* Groups elements according to key and window.
- * @param <K> key type.
+ *
+ * @param <K> key type.
* @param <InputT> input type.
*/
public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
@@ -55,12 +56,13 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* GroupByKey constructor.
- * @param outputCoders output coders
- * @param mainOutputTag main output tag
+ *
+ * @param outputCoders output coders
+ * @param mainOutputTag main output tag
* @param windowingStrategy windowing strategy
- * @param options pipeline options
- * @param reduceFn reduce function
- * @param displayData display data.
+ * @param options pipeline options
+ * @param reduceFn reduce function
+ * @param displayData display data.
*/
public GroupByKeyAndWindowDoFnTransform(final Map<TupleTag<?>, Coder<?>> outputCoders,
final TupleTag<KV<K, Iterable<InputT>>> mainOutputTag,
@@ -85,6 +87,7 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* This creates a new DoFn that groups elements by key and window.
+ *
* @param doFn original doFn.
* @return GroupAlsoByWindowViaWindowSetNewDoFn
*/
@@ -114,6 +117,7 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* It collects data for each key.
* The collected data are emitted at {@link GroupByKeyAndWindowDoFnTransform#onWatermark(Watermark)}
+ *
* @param element data element
*/
@Override
@@ -134,8 +138,9 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* Process the collected data and trigger timers.
- * @param inputWatermark current input watermark
- * @param processingTime processing time
+ *
+ * @param inputWatermark current input watermark
+ * @param processingTime processing time
* @param synchronizedTime synchronized time
*/
private void processElementsAndTriggerTimers(final Watermark inputWatermark,
@@ -166,7 +171,8 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* Output watermark
* = max(prev output watermark,
- * min(input watermark, watermark holds)).
+ * min(input watermark, watermark holds)).
+ *
* @param inputWatermark input watermark
*/
private void emitOutputWatermark(final Watermark inputWatermark) {
@@ -219,9 +225,10 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* Trigger times for current key.
* When triggering, it emits the windowed data to downstream operators.
- * @param key key
- * @param watermark watermark
- * @param processingTime processing time
+ *
+ * @param key key
+ * @param watermark watermark
+ * @param processingTime processing time
* @param synchronizedTime synchronized time
*/
private void triggerTimers(final K key,
@@ -252,6 +259,7 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
/**
* Get timer data.
+ *
* @param timerInternals in-memory timer internals.
* @return list of timer datas.
*/
@@ -382,6 +390,7 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
public void emitWatermark(final Watermark watermark) {
outputCollector.emitWatermark(watermark);
}
+
@Override
public <T> void emit(final String dstVertexId, final T output) {
outputCollector.emit(dstVertexId, output);
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyTransform.java
index 71c68ea..ab3f258 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyTransform.java
@@ -19,9 +19,9 @@
package org.apache.nemo.compiler.frontend.beam.transform;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
import org.apache.nemo.common.ir.OutputCollector;
import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
-import org.apache.beam.sdk.values.KV;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +29,7 @@ import java.util.*;
/**
* Group Beam KVs.
+ *
* @param <I> input type.
*/
public final class GroupByKeyTransform<I> extends NoWatermarkEmitTransform<I, WindowedValue<KV<Object, List>>> {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/LoopCompositeTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/LoopCompositeTransform.java
index 2c9c3b1..2f2129e 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/LoopCompositeTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/LoopCompositeTransform.java
@@ -28,9 +28,10 @@ import java.io.Serializable;
* A composite transform for wrapping transforms inside a loop to create loop operators in the IR.
* Each iterations are stateless, as the repeating iterations are zipped into a single copy.
* We assume a single {@link LoopCompositeTransform} inside a for/while loop.
- * @param <inputT> input type of the composite transform.
+ *
+ * @param <inputT> input type of the composite transform.
* @param <outputT> output type of the composite transform.
*/
public abstract class LoopCompositeTransform<inputT extends PInput, outputT extends POutput>
- extends PTransform<inputT, outputT> implements Serializable {
+ extends PTransform<inputT, outputT> implements Serializable {
}
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
index 7ed5150..a9b8572 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
@@ -40,7 +40,7 @@ import java.util.Map;
/**
* DoFn transform implementation with push backs for side inputs.
*
- * @param <InputT> input type.
+ * @param <InputT> input type.
* @param <OutputT> output type.
*/
public final class PushBackDoFnTransform<InputT, OutputT> extends AbstractDoFnTransform<InputT, InputT, OutputT> {
@@ -53,15 +53,16 @@ public final class PushBackDoFnTransform<InputT, OutputT> extends AbstractDoFnTr
/**
* PushBackDoFnTransform Constructor.
- * @param doFn doFn
- * @param inputCoder input coder
- * @param outputCoders output coders
- * @param mainOutputTag main output tag
+ *
+ * @param doFn doFn
+ * @param inputCoder input coder
+ * @param outputCoders output coders
+ * @param mainOutputTag main output tag
* @param additionalOutputTags additional output tags
- * @param windowingStrategy windowing strategy
- * @param sideInputs side inputs
- * @param options pipeline options
- * @param displayData display data.
+ * @param windowingStrategy windowing strategy
+ * @param sideInputs side inputs
+ * @param options pipeline options
+ * @param displayData display data.
*/
public PushBackDoFnTransform(final DoFn<InputT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/SideInputTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/SideInputTransform.java
index 0ddb631..4758066 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/SideInputTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/SideInputTransform.java
@@ -27,6 +27,7 @@ import org.apache.nemo.compiler.frontend.beam.SideInputElement;
/**
* Side input transform implementation.
* TODO #297: Consider Removing SideInputTransform
+ *
* @param <T> input/output type.
*/
public final class SideInputTransform<T> implements Transform<WindowedValue<T>, WindowedValue<SideInputElement<T>>> {
@@ -35,6 +36,7 @@ public final class SideInputTransform<T> implements Transform<WindowedValue<T>,
/**
* Constructor.
+ *
* @param index side input index.
*/
public SideInputTransform(final int index) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/WindowFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/WindowFnTransform.java
index a3dd2e1..14ea5ae 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/WindowFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/WindowFnTransform.java
@@ -22,10 +22,10 @@ import com.google.common.collect.Iterables;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.nemo.common.ir.OutputCollector;
import org.apache.nemo.common.ir.vertex.transform.Transform;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.nemo.common.punctuation.Watermark;
import org.joda.time.Instant;
@@ -35,6 +35,7 @@ import java.util.Collection;
* Windowing transform implementation.
* This transform simply windows the given elements into
* finite windows according to a user-specified WindowFnTransform.
+ *
* @param <T> input/output type.
* @param <W> window type
*/
@@ -46,7 +47,8 @@ public final class WindowFnTransform<T, W extends BoundedWindow>
/**
* Default Constructor.
- * @param windowFn windowFn for the Transform.
+ *
+ * @param windowFn windowFn for the Transform.
* @param displayData display data.
*/
public WindowFnTransform(final WindowFn windowFn, final DisplayData displayData) {
diff --git a/compiler/frontend/spark/pom.xml b/compiler/frontend/spark/pom.xml
index eb2c434..a40468a 100644
--- a/compiler/frontend/spark/pom.xml
+++ b/compiler/frontend/spark/pom.xml
@@ -17,121 +17,122 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler</artifactId>
- <version>0.2-SNAPSHOT</version>
- <relativePath>../../</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
- <artifactId>nemo-compiler-frontend-spark</artifactId>
- <name>Nemo Compiler Frontend: Spark</name>
+ <artifactId>nemo-compiler-frontend-spark</artifactId>
+ <name>Nemo Compiler Frontend: Spark</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-client</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.11</artifactId>
- <version>${spark.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.11</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.11</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.11</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ </dependency>
+ </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.3.1</version>
- <configuration>
- <recompileMode>incremental</recompileMode>
- <javacArgs>
- <javacArg>-Xlint:unchecked</javacArg>
- <javacArg>-Xlint:deprecation</javacArg>
- </javacArgs>
- </configuration>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.0.2</version>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
- <configuration>
- <outputDirectory>docs/apidocs</outputDirectory>
- <reportOutputDirectory>docs/apidocs</reportOutputDirectory>
- <doclint>-missing</doclint>
- </configuration>
- <executions>
- <execution>
- <id>aggregate</id>
- <goals>
- <goal>aggregate</goal>
- </goals>
- <phase>site</phase>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.3.1</version>
+ <configuration>
+ <recompileMode>incremental</recompileMode>
+ <javacArgs>
+ <javacArg>-Xlint:unchecked</javacArg>
+ <javacArg>-Xlint:deprecation</javacArg>
+ </javacArgs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>add-source</goal>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <executions>
+ <execution>
+ <phase>compile</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <outputDirectory>docs/apidocs</outputDirectory>
+ <reportOutputDirectory>docs/apidocs</reportOutputDirectory>
+ <doclint>-missing</doclint>
+ </configuration>
+ <executions>
+ <execution>
+ <id>aggregate</id>
+ <goals>
+ <goal>aggregate</goal>
+ </goals>
+ <phase>site</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkDecoderFactory.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkDecoderFactory.java
index cec71d2..9e9e8b4 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkDecoderFactory.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkDecoderFactory.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
/**
* Spark DecoderFactory for serialization.
+ *
* @param <T> type of the object to deserialize.
*/
public final class SparkDecoderFactory<T> implements DecoderFactory<T> {
@@ -56,6 +57,7 @@ public final class SparkDecoderFactory<T> implements DecoderFactory<T> {
/**
* SparkDecoder.
+ *
* @param <T2> type of the object to deserialize.
*/
private final class SparkDecoder<T2> implements Decoder<T2> {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkEncoderFactory.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkEncoderFactory.java
index a9d6d59..bb1174b 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkEncoderFactory.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/coder/SparkEncoderFactory.java
@@ -28,6 +28,7 @@ import java.io.OutputStream;
/**
* Spark EncoderFactory for serialization.
+ *
* @param <T> type of the object to serialize.
*/
public final class SparkEncoderFactory<T> implements EncoderFactory<T> {
@@ -56,6 +57,7 @@ public final class SparkEncoderFactory<T> implements EncoderFactory<T> {
/**
* SparkEncoder.
+ *
* @param <T2> type of the object to serialize.
*/
private final class SparkEncoder<T2> implements Encoder<T2> {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/JavaSparkContext.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/JavaSparkContext.java
index c667041..d4cd2a7 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/JavaSparkContext.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/JavaSparkContext.java
@@ -31,6 +31,7 @@ public final class JavaSparkContext {
/**
* Constructor.
+ *
* @param sparkContext spark context to wrap.
*/
public JavaSparkContext(final SparkContext sparkContext) {
@@ -62,7 +63,7 @@ public final class JavaSparkContext {
* Initiate a JavaRDD with the number of parallelism.
*
* @param list input data as list.
- * @param <T> type of the initial element.
+ * @param <T> type of the initial element.
* @return the newly initiated JavaRDD.
*/
public <T> JavaRDD<T> parallelize(final List<T> list) {
@@ -72,9 +73,9 @@ public final class JavaSparkContext {
/**
* Initiate a JavaRDD with the number of parallelism.
*
- * @param l input data as list.
+ * @param l input data as list.
* @param slices number of slices (parallelism).
- * @param <T> type of the initial element.
+ * @param <T> type of the initial element.
* @return the newly initiated JavaRDD.
*/
public <T> JavaRDD<T> parallelize(final List<T> l, final int slices) {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkBroadcast.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkBroadcast.java
index c344b5b..e267873 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkBroadcast.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkBroadcast.java
@@ -29,7 +29,8 @@ public final class SparkBroadcast<T> extends org.apache.spark.broadcast.Broadcas
/**
* Constructor.
- * @param tag broadcast id.
+ *
+ * @param tag broadcast id.
* @param classType class type.
*/
SparkBroadcast(final long tag, final Class<T> classType) {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkContext.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkContext.java
index 5d69470..c12b3b0 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkContext.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkContext.java
@@ -57,9 +57,9 @@ public final class SparkContext extends org.apache.spark.SparkContext {
/**
* Initiate a JavaRDD with the number of parallelism.
*
- * @param seq input data as list.
- * @param numSlices number of slices (parallelism).
- * @param evidence type of the initial element.
+ * @param seq input data as list.
+ * @param numSlices number of slices (parallelism).
+ * @param evidence type of the initial element.
* @return the newly initiated JavaRDD.
*/
@Override
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java
index 680a29b..2f15d54 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java
@@ -41,7 +41,10 @@ import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.serializer.*;
+import org.apache.spark.serializer.JavaSerializer;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.serializer.Serializer;
+import org.apache.spark.serializer.SerializerInstance;
import scala.Function1;
import scala.Tuple2;
import scala.collection.JavaConverters;
@@ -50,7 +53,9 @@ import scala.reflect.ClassTag;
import scala.reflect.ClassTag$;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Stack;
/**
* Utility class for RDDs.
@@ -72,7 +77,7 @@ public final class SparkFrontendUtils {
*/
public static Serializer deriveSerializerFrom(final org.apache.spark.SparkContext sparkContext) {
if (sparkContext.conf().get("spark.serializer", "")
- .equals("org.apache.spark.serializer.KryoSerializer")) {
+ .equals("org.apache.spark.serializer.KryoSerializer")) {
return new KryoSerializer(sparkContext.conf());
} else {
return new JavaSerializer(sparkContext.conf());
@@ -99,7 +104,7 @@ public final class SparkFrontendUtils {
builder.addVertex(collectVertex, loopVertexStack);
final IREdge newEdge = new IREdge(getEdgeCommunicationPattern(lastVertex, collectVertex),
- lastVertex, collectVertex);
+ lastVertex, collectVertex);
newEdge.setProperty(EncoderProperty.of(new SparkEncoderFactory(serializer)));
newEdge.setProperty(DecoderProperty.of(new SparkDecoderFactory(serializer)));
newEdge.setProperty(SPARK_KEY_EXTRACTOR_PROP);
@@ -121,8 +126,8 @@ public final class SparkFrontendUtils {
public static CommunicationPatternProperty.Value getEdgeCommunicationPattern(final IRVertex src,
final IRVertex dst) {
if (dst instanceof OperatorVertex
- && (((OperatorVertex) dst).getTransform() instanceof ReduceByKeyTransform
- || ((OperatorVertex) dst).getTransform() instanceof GroupByKeyTransform)) {
+ && (((OperatorVertex) dst).getTransform() instanceof ReduceByKeyTransform
+ || ((OperatorVertex) dst).getTransform() instanceof GroupByKeyTransform)) {
return CommunicationPatternProperty.Value.Shuffle;
} else {
return CommunicationPatternProperty.Value.OneToOne;
@@ -131,7 +136,7 @@ public final class SparkFrontendUtils {
/**
* Converts a {@link Function1} to a corresponding {@link Function}.
- *
+ * <p>
* Here, we use the Spark 'JavaSerializer' to facilitate debugging in the future.
* TODO #205: RDD Closure with Broadcast Variables Serialization Bug
*
@@ -189,7 +194,7 @@ public final class SparkFrontendUtils {
* @return the converted Java function.
*/
public static <I, O> FlatMapFunction<I, O> toJavaFlatMapFunction(
- final Function1<I, TraversableOnce<O>> scalaFunction) {
+ final Function1<I, TraversableOnce<O>> scalaFunction) {
return new FlatMapFunction<I, O>() {
@Override
public Iterator<O> call(final I i) throws Exception {
@@ -208,7 +213,7 @@ public final class SparkFrontendUtils {
* @return the converted map function.
*/
public static <T, K, V> Function<T, Tuple2<K, V>> pairFunctionToPlainFunction(
- final PairFunction<T, K, V> pairFunction) {
+ final PairFunction<T, K, V> pairFunction) {
return new Function<T, Tuple2<K, V>>() {
@Override
public Tuple2<K, V> call(final T elem) throws Exception {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaPairRDD.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaPairRDD.java
index 4ef5999..1c85965 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaPairRDD.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaPairRDD.java
@@ -39,6 +39,7 @@ import java.util.Map;
/**
* Java RDD for pairs.
+ *
* @param <K> key type.
* @param <V> value type.
*/
@@ -95,7 +96,7 @@ public final class JavaPairRDD<K, V> extends org.apache.spark.api.java.JavaPairR
public JavaPairRDD<K, V> reduceByKey(final Function2<V, V, V> func) {
// Explicit conversion
final PairRDDFunctions<K, V> pairRdd = RDD.rddToPairRDDFunctions(
- rdd, ClassTag$.MODULE$.apply(Object.class), ClassTag$.MODULE$.apply(Object.class), null);
+ rdd, ClassTag$.MODULE$.apply(Object.class), ClassTag$.MODULE$.apply(Object.class), null);
final RDD<Tuple2<K, V>> reducedRdd = pairRdd.reduceByKey(func);
return JavaPairRDD.fromRDD(reducedRdd);
}
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaRDD.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaRDD.java
index 18302da..e8b8f12 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaRDD.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/core/rdd/JavaRDD.java
@@ -29,7 +29,10 @@ import org.apache.nemo.compiler.frontend.spark.source.SparkDatasetBoundedSourceV
import org.apache.nemo.compiler.frontend.spark.source.SparkTextFileBoundedSourceVertex;
import org.apache.nemo.compiler.frontend.spark.sql.Dataset;
import org.apache.nemo.compiler.frontend.spark.sql.SparkSession;
-import org.apache.spark.*;
+import org.apache.spark.Partition;
+import org.apache.spark.Partitioner;
+import org.apache.spark.SparkContext;
+import org.apache.spark.TaskContext;
import org.apache.spark.api.java.JavaFutureAction;
import org.apache.spark.api.java.Optional;
import org.apache.spark.api.java.function.*;
@@ -40,11 +43,15 @@ import scala.Option;
import scala.Tuple2;
import scala.reflect.ClassTag$;
-import java.util.*;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Java RDD.
+ *
* @param <T> type of the final element.
*/
public final class JavaRDD<T> extends org.apache.spark.api.java.JavaRDD<T> {
@@ -70,7 +77,7 @@ public final class JavaRDD<T> extends org.apache.spark.api.java.JavaRDD<T> {
builder.addVertex(initializedSourceVertex);
final RDD<T> nemoRdd = new RDD<>(sparkContext, builder.buildWithoutSourceSinkCheck(),
- initializedSourceVertex, Option.empty(), ClassTag$.MODULE$.apply(Object.class));
+ initializedSourceVertex, Option.empty(), ClassTag$.MODULE$.apply(Object.class));
return new JavaRDD<>(nemoRdd);
}
@@ -115,7 +122,7 @@ public final class JavaRDD<T> extends org.apache.spark.api.java.JavaRDD<T> {
builder.addVertex(sparkBoundedSourceVertex);
return new JavaRDD<>(
- sparkRDD, sparkSession.sparkContext(), builder.buildWithoutSourceSinkCheck(), sparkBoundedSourceVertex);
+ sparkRDD, sparkSession.sparkContext(), builder.buildWithoutSourceSinkCheck(), sparkBoundedSourceVertex);
}
/**
@@ -203,7 +210,7 @@ public final class JavaRDD<T> extends org.apache.spark.api.java.JavaRDD<T> {
@Override
public <K2, V2> JavaPairRDD<K2, V2> mapToPair(final PairFunction<T, K2, V2> f) {
final RDD<Tuple2<K2, V2>> pairRdd =
- rdd.map(SparkFrontendUtils.pairFunctionToPlainFunction(f), ClassTag$.MODULE$.apply(Object.class));
+ rdd.map(SparkFrontendUtils.pairFunctionToPlainFunction(f), ClassTag$.MODULE$.apply(Object.class));
return JavaPairRDD.fromRDD(pairRdd);
}
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkDatasetBoundedSourceVertex.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkDatasetBoundedSourceVertex.java
index d7b3794..db74eca 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkDatasetBoundedSourceVertex.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkDatasetBoundedSourceVertex.java
@@ -23,7 +23,8 @@ import org.apache.nemo.common.ir.Readable;
import org.apache.nemo.common.ir.vertex.SourceVertex;
import org.apache.nemo.compiler.frontend.spark.sql.Dataset;
import org.apache.nemo.compiler.frontend.spark.sql.SparkSession;
-import org.apache.spark.*;
+import org.apache.spark.Partition;
+import org.apache.spark.TaskContext$;
import org.apache.spark.rdd.RDD;
import scala.collection.JavaConverters;
@@ -33,6 +34,7 @@ import java.util.*;
/**
* Bounded source vertex for Spark Dataset.
+ *
* @param <T> type of data to read.
*/
public final class SparkDatasetBoundedSourceVertex<T> extends SourceVertex<T> {
@@ -51,10 +53,10 @@ public final class SparkDatasetBoundedSourceVertex<T> extends SourceVertex<T> {
final Partition[] partitions = rdd.getPartitions();
for (int i = 0; i < partitions.length; i++) {
readables.add(new SparkDatasetBoundedSourceReadable(
- partitions[i],
- sparkSession.getDatasetCommandsList(),
- sparkSession.getInitialConf(),
- i));
+ partitions[i],
+ sparkSession.getDatasetCommandsList(),
+ sparkSession.getInitialConf(),
+ i));
}
this.estimatedByteSize = dataset.javaRDD()
.map(o -> (long) o.toString().getBytes("UTF-8").length)
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkTextFileBoundedSourceVertex.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkTextFileBoundedSourceVertex.java
index 5679c67..500162c 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkTextFileBoundedSourceVertex.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/source/SparkTextFileBoundedSourceVertex.java
@@ -21,13 +21,18 @@ package org.apache.nemo.compiler.frontend.spark.source;
import org.apache.nemo.common.ir.BoundedIteratorReadable;
import org.apache.nemo.common.ir.Readable;
import org.apache.nemo.common.ir.vertex.SourceVertex;
-import org.apache.spark.*;
+import org.apache.spark.Partition;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.TaskContext$;
import org.apache.spark.rdd.RDD;
import org.apache.spark.util.SizeEstimator;
import scala.collection.JavaConverters;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
/**
* Bounded source vertex for Spark text file.
@@ -50,11 +55,11 @@ public final class SparkTextFileBoundedSourceVertex extends SourceVertex<String>
final Partition[] partitions = sparkContext.textFile(inputPath, numPartitions).getPartitions();
for (int i = 0; i < partitions.length; i++) {
readables.add(new SparkTextFileBoundedSourceReadable(
- partitions[i],
- sparkContext.getConf(),
- i,
- inputPath,
- numPartitions));
+ partitions[i],
+ sparkContext.getConf(),
+ i,
+ inputPath,
+ numPartitions));
}
this.estimatedSizeBytes = SizeEstimator.estimate(sparkContext.textFile(inputPath, numPartitions));
}
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/DataFrameReader.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/DataFrameReader.java
index ff9bdbc..5de3da8 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/DataFrameReader.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/DataFrameReader.java
@@ -31,6 +31,7 @@ public final class DataFrameReader extends org.apache.spark.sql.DataFrameReader
/**
* Constructor.
+ *
* @param sparkSession spark session.
*/
DataFrameReader(final SparkSession sparkSession) {
@@ -103,9 +104,9 @@ public final class DataFrameReader extends org.apache.spark.sql.DataFrameReader
final long lowerBound, final long upperBound, final int numPartitions,
final java.util.Properties connectionProperties) {
final boolean userTriggered = initializeFunction(
- url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties);
+ url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties);
final Dataset<Row> result = Dataset.from(super.jdbc(
- url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties));
+ url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties));
this.setIsUserTriggered(userTriggered);
return result;
}
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/Dataset.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/Dataset.java
index 59ab5ea..7eb999e 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/Dataset.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/Dataset.java
@@ -33,6 +33,7 @@ import java.util.stream.Stream;
/**
* A dataset component: it represents relational data.
+ *
* @param <T> type of the data.
*/
public final class Dataset<T> extends org.apache.spark.sql.Dataset<T> implements NemoSparkUserFacingClass {
@@ -90,18 +91,18 @@ public final class Dataset<T> extends org.apache.spark.sql.Dataset<T> implements
/**
* Create a {@link RDD} component from this data set.
* To transparently give our RDD to user programs, this method have to be overridden.
- *
+ * <p>
* By overriding this method, if a method (such as reduce) of super ({@link org.apache.spark.sql.Dataset}) is called
* and it uses super's rdd, the rdd will be our rdd returned by this method.
* This is an intended behavior and the result will be calculated by our system.
*
* @return the new RDD component.
*/
- @Override
- public RDD<T> rdd() {
- final JavaRDD<T> javaRDD = JavaRDD.of((SparkSession) super.sparkSession(), this);
- return javaRDD.rdd();
- }
+ @Override
+ public RDD<T> rdd() {
+ final JavaRDD<T> javaRDD = JavaRDD.of((SparkSession) super.sparkSession(), this);
+ return javaRDD.rdd();
+ }
@Override
public Dataset<Row> agg(final Column expr, final Column... exprs) {
@@ -444,8 +445,8 @@ public final class Dataset<T> extends org.apache.spark.sql.Dataset<T> implements
@Override
public <U> Dataset<U> mapPartitions(
- final scala.Function1<scala.collection.Iterator<T>, scala.collection.Iterator<U>> func,
- final Encoder<U> evidence) {
+ final scala.Function1<scala.collection.Iterator<T>, scala.collection.Iterator<U>> func,
+ final Encoder<U> evidence) {
final boolean userTriggered = initializeFunction(func, evidence);
final Dataset<U> result = from(super.mapPartitions(func, evidence));
this.setIsUserTriggered(userTriggered);
@@ -536,10 +537,10 @@ public final class Dataset<T> extends org.apache.spark.sql.Dataset<T> implements
return result;
}
-// @Override
-// public java.util.List<Dataset<T>> randomSplitAsList(double[] weights, long seed) {
-// return super.randomSplitAsList(weights, seed).stream().map(ds -> from(ds)).collect(Collectors.toList());
-// }
+ // @Override
+ // public java.util.List<Dataset<T>> randomSplitAsList(double[] weights, long seed) {
+ // return super.randomSplitAsList(weights, seed).stream().map(ds -> from(ds)).collect(Collectors.toList());
+ // }
@Override
public Dataset<T> repartition(final Column... partitionExprs) {
@@ -756,7 +757,7 @@ public final class Dataset<T> extends org.apache.spark.sql.Dataset<T> implements
@Override
public <U> Dataset<U> transform(
- final scala.Function1<org.apache.spark.sql.Dataset<T>, org.apache.spark.sql.Dataset<U>> t) {
+ final scala.Function1<org.apache.spark.sql.Dataset<T>, org.apache.spark.sql.Dataset<U>> t) {
final boolean userTriggered = initializeFunction(t);
final Dataset<U> result = from(super.transform(t));
this.setIsUserTriggered(userTriggered);
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/NemoSparkUserFacingClass.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/NemoSparkUserFacingClass.java
index fe0fd1a..8cf3b82 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/NemoSparkUserFacingClass.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/NemoSparkUserFacingClass.java
@@ -32,6 +32,7 @@ public interface NemoSparkUserFacingClass {
/**
* Set the userTriggered flag.
+ *
* @param bool boolean to set the flag to.
*/
default void setIsUserTriggered(boolean bool) {
@@ -45,6 +46,7 @@ public interface NemoSparkUserFacingClass {
/**
* A method to distinguish user-called functions from internal calls.
+ *
* @param args arguments of the method
* @return whether or not this function has been called by the user.
*/
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java
index 17aeecd..9c6fbec 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java
@@ -18,10 +18,10 @@
*/
package org.apache.nemo.compiler.frontend.spark.sql;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.compiler.frontend.spark.core.SparkContext;
import org.apache.nemo.conf.JobConf;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.reef.tang.Configuration;
import org.apache.reef.tang.Injector;
import org.apache.reef.tang.Tang;
@@ -115,7 +115,7 @@ public final class SparkSession extends org.apache.spark.sql.SparkSession implem
*/
public static <T> Dataset<T> initializeDataset(final SparkSession spark,
final LinkedHashMap<String, Object[]> commandList)
- throws OperationNotSupportedException {
+ throws OperationNotSupportedException {
Object result = spark;
for (Map.Entry<String, Object[]> command : commandList.entrySet()) {
@@ -126,8 +126,8 @@ public final class SparkSession extends org.apache.spark.sql.SparkSession implem
final Class<?>[] argTypes = Stream.of(args).map(Object::getClass).toArray(Class[]::new);
if (!className.contains("SparkSession")
- && !className.contains("DataFrameReader")
- && !className.contains("Dataset")) {
+ && !className.contains("DataFrameReader")
+ && !className.contains("Dataset")) {
throw new OperationNotSupportedException(command + " is not yet supported.");
}
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java
index 7f5aad6..08d8af6 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java
@@ -18,9 +18,9 @@
*/
package org.apache.nemo.compiler.frontend.spark.transform;
+import org.apache.commons.lang3.SerializationUtils;
import org.apache.nemo.common.ir.OutputCollector;
import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
-import org.apache.commons.lang3.SerializationUtils;
import java.util.ArrayList;
import java.util.Base64;
@@ -28,6 +28,7 @@ import java.util.Base64;
/**
* Collect transform.
* This transform is used for spark batch job, so do not emit watermark.
+ *
* @param <T> type of data to collect.
*/
public final class CollectTransform<T> extends NoWatermarkEmitTransform<T, T> {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/FlatMapTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/FlatMapTransform.java
index aca41a0..a0ded0e 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/FlatMapTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/FlatMapTransform.java
@@ -25,6 +25,7 @@ import org.apache.spark.api.java.function.FlatMapFunction;
/**
* Flatmap Transform that flattens each output element after mapping each elements to an iterator.
+ *
* @param <T> input type.
* @param <U> output type.
*/
@@ -34,6 +35,7 @@ public final class FlatMapTransform<T, U> implements Transform<T, U> {
/**
* Constructor.
+ *
* @param func flat map function.
*/
public FlatMapTransform(final FlatMapFunction<T, U> func) {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/GroupByKeyTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/GroupByKeyTransform.java
index 7d39aaf..3c36ba6 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/GroupByKeyTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/GroupByKeyTransform.java
@@ -23,10 +23,14 @@ import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
import org.apache.nemo.common.ir.vertex.transform.Transform;
import scala.Tuple2;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* Transform for group by key transformation.
+ *
* @param <K> key type.
* @param <V> value type.
*/
@@ -58,7 +62,7 @@ public final class GroupByKeyTransform<K, V> extends NoWatermarkEmitTransform<Tu
@Override
public void close() {
keyToValues.entrySet().stream().map(entry -> new Tuple2<>(entry.getKey(), (Iterable<V>) entry.getValue()))
- .forEach(outputCollector::emit);
+ .forEach(outputCollector::emit);
keyToValues.clear();
}
}
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/HDFSTextFileTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/HDFSTextFileTransform.java
index cf5f14d..6822498 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/HDFSTextFileTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/HDFSTextFileTransform.java
@@ -18,13 +18,13 @@
*/
package org.apache.nemo.compiler.frontend.spark.transform;
-import org.apache.nemo.common.ir.OutputCollector;
-import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
-import org.apache.nemo.common.ir.vertex.transform.Transform;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.nemo.common.ir.OutputCollector;
+import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
+import org.apache.nemo.common.ir.vertex.transform.Transform;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,6 +33,7 @@ import java.util.UUID;
/**
* Transform which saves elements to a HDFS text file for Spark.
+ *
* @param <I> input type.
*/
public final class HDFSTextFileTransform<I> extends NoWatermarkEmitTransform<I, String> {
@@ -63,8 +64,8 @@ public final class HDFSTextFileTransform<I> extends NoWatermarkEmitTransform<I,
@Override
public void close() {
try (
- final FileSystem fileSystem = fileName.getFileSystem(new JobConf());
- final FSDataOutputStream outputStream = fileSystem.create(fileName, false);
+ final FileSystem fileSystem = fileName.getFileSystem(new JobConf());
+ final FSDataOutputStream outputStream = fileSystem.create(fileName, false);
) {
for (final I element : elements) {
outputStream.writeBytes(element + "\n");
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/LocalTextFileTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/LocalTextFileTransform.java
index 8f6cc8b..1de8151 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/LocalTextFileTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/LocalTextFileTransform.java
@@ -28,6 +28,7 @@ import java.util.UUID;
/**
* Transform which saves elements to a local text file for Spark.
+ *
* @param <I> input type.
*/
public final class LocalTextFileTransform<I> extends NoWatermarkEmitTransform<I, String> {
@@ -58,8 +59,8 @@ public final class LocalTextFileTransform<I> extends NoWatermarkEmitTransform<I,
@Override
public void close() {
try (
- final Writer writer =
- new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName, false), "utf-8"))
+ final Writer writer =
+ new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName, false), "utf-8"))
) {
for (final I element : elements) {
writer.write(element + "\n");
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapToPairTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapToPairTransform.java
index 32f3d12..4745574 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapToPairTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapToPairTransform.java
@@ -26,6 +26,7 @@ import scala.Tuple2;
/**
* Map elements to Pair elements.
+ *
* @param <T> input type.
* @param <K> output key type.
* @param <V> output value type.
@@ -36,6 +37,7 @@ public final class MapToPairTransform<T, K, V> implements Transform<T, Tuple2<K,
/**
* Constructor.
+ *
* @param func Pair function to apply to each element.
*/
public MapToPairTransform(final PairFunction<T, K, V> func) {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapTransform.java
index 0774b2c..177fc5c 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/MapTransform.java
@@ -25,6 +25,7 @@ import org.apache.spark.api.java.function.Function;
/**
* Map Transform for Spark.
+ *
* @param <I> input type.
* @param <O> output type.
*/
@@ -34,6 +35,7 @@ public final class MapTransform<I, O> implements Transform<I, O> {
/**
* Constructor.
+ *
* @param func the function to run map with.
*/
public MapTransform(final Function<I, O> func) {
@@ -47,11 +49,11 @@ public final class MapTransform<I, O> implements Transform<I, O> {
@Override
public void onData(final I element) {
- try {
- outputCollector.emit(func.call(element));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ try {
+ outputCollector.emit(func.call(element));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
@Override
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceByKeyTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceByKeyTransform.java
index bea19d3..9a66d72 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceByKeyTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceByKeyTransform.java
@@ -25,10 +25,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* Transform for reduce by key transformation.
+ *
* @param <K> key type.
* @param <V> value type.
*/
@@ -41,6 +45,7 @@ public final class ReduceByKeyTransform<K, V> extends NoWatermarkEmitTransform<T
/**
* Constructor.
+ *
* @param func reduce function.
*/
public ReduceByKeyTransform(final Function2<V, V, V> func) {
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceTransform.java
index 2141017..6dbe172 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/ReduceTransform.java
@@ -38,6 +38,7 @@ public final class ReduceTransform<T> implements Transform<T, T> {
/**
* Constructor.
+ *
* @param func function to run for the reduce transform.
*/
public ReduceTransform(final Function2<T, T, T> func) {
@@ -76,9 +77,10 @@ public final class ReduceTransform<T> implements Transform<T, T> {
/**
* Reduce the iterator elements into a single object.
+ *
* @param elements the iterator of elements.
- * @param func function to apply for reduction.
- * @param <T> type of the elements.
+ * @param func function to apply for reduction.
+ * @param <T> type of the elements.
* @return the reduced element.
*/
@Nullable
diff --git a/compiler/optimizer/pom.xml b/compiler/optimizer/pom.xml
index 44dca09..fb09507 100644
--- a/compiler/optimizer/pom.xml
+++ b/compiler/optimizer/pom.xml
@@ -17,44 +17,45 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler</artifactId>
- <version>0.2-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
- <artifactId>nemo-compiler-optimizer</artifactId>
- <name>Nemo Compiler Optimizer</name>
+ <artifactId>nemo-compiler-optimizer</artifactId>
+ <name>Nemo Compiler Optimizer</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>${commons-math.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-conf</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>${commons-math.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-conf</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ </dependencies>
</project>
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/NemoOptimizer.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/NemoOptimizer.java
index 3db9529..1b5ab48 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/NemoOptimizer.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/NemoOptimizer.java
@@ -53,7 +53,7 @@ public final class NemoOptimizer implements Optimizer {
/**
* @param dagDirectory to store JSON representation of intermediate DAGs.
- * @param policyName the name of the optimization policy.
+ * @param policyName the name of the optimization policy.
*/
@Inject
private NemoOptimizer(@Parameter(JobConf.DAGDirectory.class) final String dagDirectory,
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/Optimizer.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/Optimizer.java
index 4f4e142..be5f149 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/Optimizer.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/Optimizer.java
@@ -40,7 +40,7 @@ public interface Optimizer {
/**
* Optimize the submitted DAG at run time.
*
- * @param dag input.
+ * @param dag input.
* @param message for optimization.
* @return optimized DAG.
*/
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
index ec533ad..c7e5f8f 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
@@ -45,6 +45,7 @@ public final class MapReduceDisaggregationOptimization {
/**
* Main function of the example MR program.
+ *
* @param args arguments.
* @throws Exception Exceptions on the way.
*/
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/CompileTimePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/CompileTimePass.java
index 5f5dbb7..1f77e17 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/CompileTimePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/CompileTimePass.java
@@ -32,6 +32,7 @@ import java.util.function.Function;
public abstract class CompileTimePass extends Pass implements Function<IRDAG, IRDAG> {
/**
* Getter for prerequisite execution properties.
+ *
* @return set of prerequisite execution properties.
*/
public abstract Set<Class<? extends ExecutionProperty>> getPrerequisiteExecutionProperties();
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/AnnotatingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/AnnotatingPass.java
index e32415a..a33053a 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/AnnotatingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/AnnotatingPass.java
@@ -36,6 +36,7 @@ public abstract class AnnotatingPass extends CompileTimePass {
/**
* Constructor.
+ *
* @param cls the annotating pass class.
*/
public AnnotatingPass(final Class<? extends AnnotatingPass> cls) {
@@ -44,11 +45,12 @@ public abstract class AnnotatingPass extends CompileTimePass {
final Requires requires = cls.getAnnotation(Requires.class);
this.prerequisiteExecutionProperties = requires == null
- ? new HashSet<>() : new HashSet<>(Arrays.asList(requires.value()));
+ ? new HashSet<>() : new HashSet<>(Arrays.asList(requires.value()));
}
/**
* Getter for the execution properties to annotate through the pass.
+ *
* @return key of execution properties to annotate through the pass.
*/
public final Set<Class<? extends ExecutionProperty>> getExecutionPropertiesToAnnotate() {
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/CompressionPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/CompressionPass.java
index b8c1b18..e4069b7 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/CompressionPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/CompressionPass.java
@@ -39,6 +39,7 @@ public final class CompressionPass extends AnnotatingPass {
/**
* Constructor.
+ *
* @param compression Compression to apply on edges.
*/
public CompressionPass(final CompressionProperty.Value compression) {
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataPersistencePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataPersistencePass.java
index 1a947f4..17d0f50 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataPersistencePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataPersistencePass.java
@@ -40,18 +40,18 @@ public final class DefaultDataPersistencePass extends AnnotatingPass {
@Override
public IRDAG apply(final IRDAG dag) {
dag.topologicalDo(irVertex ->
- dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> {
- if (!irEdge.getPropertyValue(DataPersistenceProperty.class).isPresent()) {
- final DataStoreProperty.Value dataStoreValue
- = irEdge.getPropertyValue(DataStoreProperty.class).get();
- if (DataStoreProperty.Value.MemoryStore.equals(dataStoreValue)
- || DataStoreProperty.Value.SerializedMemoryStore.equals(dataStoreValue)) {
- irEdge.setProperty(DataPersistenceProperty.of(DataPersistenceProperty.Value.Discard));
- } else {
- irEdge.setProperty(DataPersistenceProperty.of(DataPersistenceProperty.Value.Keep));
- }
+ dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> {
+ if (!irEdge.getPropertyValue(DataPersistenceProperty.class).isPresent()) {
+ final DataStoreProperty.Value dataStoreValue
+ = irEdge.getPropertyValue(DataStoreProperty.class).get();
+ if (DataStoreProperty.Value.MemoryStore.equals(dataStoreValue)
+ || DataStoreProperty.Value.SerializedMemoryStore.equals(dataStoreValue)) {
+ irEdge.setProperty(DataPersistenceProperty.of(DataPersistenceProperty.Value.Discard));
+ } else {
+ irEdge.setProperty(DataPersistenceProperty.of(DataPersistenceProperty.Value.Keep));
}
- }));
+ }
+ }));
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataStorePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataStorePass.java
index f8ac7e6..4d179d0 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataStorePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultDataStorePass.java
@@ -37,9 +37,9 @@ public final class DefaultDataStorePass extends AnnotatingPass {
public IRDAG apply(final IRDAG dag) {
dag.getVertices().forEach(vertex -> {
dag.getIncomingEdgesOf(vertex).stream()
- .filter(edge -> !edge.getPropertyValue(DataStoreProperty.class).isPresent())
- .forEach(edge -> edge.setProperty(
- DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)));
+ .filter(edge -> !edge.getPropertyValue(DataStoreProperty.class).isPresent())
+ .forEach(edge -> edge.setProperty(
+ DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)));
});
return dag;
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeDecoderPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeDecoderPass.java
index e513065..53a877c 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeDecoderPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeDecoderPass.java
@@ -29,7 +29,7 @@ import org.apache.nemo.common.ir.edge.executionproperty.DecoderProperty;
public final class DefaultEdgeDecoderPass extends AnnotatingPass {
private static final DecoderProperty DEFAULT_DECODER_PROPERTY =
- DecoderProperty.of(DecoderFactory.DUMMY_DECODER_FACTORY);
+ DecoderProperty.of(DecoderFactory.DUMMY_DECODER_FACTORY);
/**
* Default constructor.
@@ -41,11 +41,11 @@ public final class DefaultEdgeDecoderPass extends AnnotatingPass {
@Override
public IRDAG apply(final IRDAG dag) {
dag.topologicalDo(irVertex ->
- dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> {
- if (!irEdge.getPropertyValue(DecoderProperty.class).isPresent()) {
- irEdge.setProperty(DEFAULT_DECODER_PROPERTY);
- }
- }));
+ dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> {
+ if (!irEdge.getPropertyValue(DecoderProperty.class).isPresent()) {
+ irEdge.setProperty(DEFAULT_DECODER_PROPERTY);
+ }
+ }));
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeEncoderPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeEncoderPass.java
index 790391c..030decb 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeEncoderPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeEncoderPass.java
@@ -29,7 +29,7 @@ import org.apache.nemo.common.ir.edge.executionproperty.EncoderProperty;
public final class DefaultEdgeEncoderPass extends AnnotatingPass {
private static final EncoderProperty DEFAULT_DECODER_PROPERTY =
- EncoderProperty.of(EncoderFactory.DUMMY_ENCODER_FACTORY);
+ EncoderProperty.of(EncoderFactory.DUMMY_ENCODER_FACTORY);
/**
* Default constructor.
@@ -41,11 +41,11 @@ public final class DefaultEdgeEncoderPass extends AnnotatingPass {
@Override
public IRDAG apply(final IRDAG dag) {
dag.topologicalDo(irVertex ->
- dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> {
- if (!irEdge.getPropertyValue(EncoderProperty.class).isPresent()) {
- irEdge.setProperty(DEFAULT_DECODER_PROPERTY);
- }
- }));
+ dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> {
+ if (!irEdge.getPropertyValue(EncoderProperty.class).isPresent()) {
+ irEdge.setProperty(DEFAULT_DECODER_PROPERTY);
+ }
+ }));
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPass.java
index 614c53f..0d45249 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPass.java
@@ -74,22 +74,22 @@ public final class DefaultParallelismPass extends AnnotatingPass {
// We manipulate them if it is set as default value of 1.
if (!originalParallelism.isPresent()) {
vertex.setProperty(ParallelismProperty.of(
- sourceVertex.getReadables(desiredSourceParallelism).size()));
+ sourceVertex.getReadables(desiredSourceParallelism).size()));
}
} else if (!inEdges.isEmpty()) {
// No reason to propagate via Broadcast edges, as the data streams that will use the broadcasted data
// as a sideInput will have their own number of parallelism
final Integer o2oParallelism = inEdges.stream()
- .filter(edge -> CommunicationPatternProperty.Value.OneToOne
- .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get()))
- .mapToInt(edge -> edge.getSrc().getPropertyValue(ParallelismProperty.class).get())
- .max().orElse(1);
+ .filter(edge -> CommunicationPatternProperty.Value.OneToOne
+ .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get()))
+ .mapToInt(edge -> edge.getSrc().getPropertyValue(ParallelismProperty.class).get())
+ .max().orElse(1);
final Integer shuffleParallelism = inEdges.stream()
- .filter(edge -> CommunicationPatternProperty.Value.Shuffle
- .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get()))
- .mapToInt(edge -> edge.getSrc().getPropertyValue(ParallelismProperty.class).get())
- .map(i -> i / shuffleDecreaseFactor)
- .max().orElse(1);
+ .filter(edge -> CommunicationPatternProperty.Value.Shuffle
+ .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get()))
+ .mapToInt(edge -> edge.getSrc().getPropertyValue(ParallelismProperty.class).get())
+ .map(i -> i / shuffleDecreaseFactor)
+ .max().orElse(1);
// We set the greater value as the parallelism.
final Integer parallelism = o2oParallelism > shuffleParallelism ? o2oParallelism : shuffleParallelism;
vertex.setProperty(ParallelismProperty.of(parallelism));
@@ -97,7 +97,7 @@ public final class DefaultParallelismPass extends AnnotatingPass {
recursivelySynchronizeO2OParallelism(dag, vertex, parallelism);
} else if (!vertex.getPropertyValue(ParallelismProperty.class).isPresent()) {
throw new RuntimeException("There is a non-source vertex that doesn't have any inEdges "
- + "(excluding SideInput edges)");
+ + "(excluding SideInput edges)");
} // No problem otherwise.
} catch (Exception e) {
throw new RuntimeException(e);
@@ -108,8 +108,9 @@ public final class DefaultParallelismPass extends AnnotatingPass {
/**
* Recursively synchronize parallelism for vertices connected by one-to-one edges.
- * @param dag the original DAG.
- * @param vertex vertex to observe and update.
+ *
+ * @param dag the original DAG.
+ * @param vertex vertex to observe and update.
* @param parallelism the parallelism of the most recently updated descendant.
* @return the max value of parallelism among those observed.
*/
@@ -117,11 +118,11 @@ public final class DefaultParallelismPass extends AnnotatingPass {
final Integer parallelism) {
final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex);
final Integer ancestorParallelism = inEdges.stream()
- .filter(edge -> CommunicationPatternProperty.Value.OneToOne
- .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get()))
- .map(IREdge::getSrc)
- .mapToInt(inVertex -> recursivelySynchronizeO2OParallelism(dag, inVertex, parallelism))
- .max().orElse(1);
+ .filter(edge -> CommunicationPatternProperty.Value.OneToOne
+ .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get()))
+ .map(IREdge::getSrc)
+ .mapToInt(inVertex -> recursivelySynchronizeO2OParallelism(dag, inVertex, parallelism))
+ .max().orElse(1);
final Integer maxParallelism = ancestorParallelism > parallelism ? ancestorParallelism : parallelism;
final Integer myParallelism = vertex.getPropertyValue(ParallelismProperty.class).get();
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPass.java
index 29dfca0..e1800d4 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPass.java
@@ -42,24 +42,24 @@ import java.util.stream.Collectors;
/**
* A pass for assigning each stages in schedule groups.
- *
+ * <p>
* TODO #347: IRDAG#partitionAcyclically
* This code can be greatly simplified...
*
* <h3>Rules</h3>
* <ul>
- * <li>Vertices connected with push edges must be assigned same ScheduleGroup.</li>
- * <li>For pull edges,
- * <ul>
- * <li>if the destination of the edge depends on multiple ScheduleGroups, split ScheduleGroup by the edge.</li>
- * <li>if the edge is broadcast type and {@code allowBroadcastWithinScheduleGroup} is {@code false},
- * split ScheduleGroup by the edge.</li>
- * <li>if the edge is shuffle type and {@code allowShuffleWithinScheduleGroup} is {@code false},
- * split ScheduleGroup by the edge.</li>
- * <li>if the destination of the edge has multiple inEdges, split ScheduleGroup by the edge.</li>
- * <li>Otherwise, the source and the destination of the edge should be assigned same ScheduleGroup.</li>
- * </ul>
- * </li>
+ * <li>Vertices connected with push edges must be assigned same ScheduleGroup.</li>
+ * <li>For pull edges,
+ * <ul>
+ * <li>if the destination of the edge depends on multiple ScheduleGroups, split ScheduleGroup by the edge.</li>
+ * <li>if the edge is broadcast type and {@code allowBroadcastWithinScheduleGroup} is {@code false},
+ * split ScheduleGroup by the edge.</li>
+ * <li>if the edge is shuffle type and {@code allowShuffleWithinScheduleGroup} is {@code false},
+ * split ScheduleGroup by the edge.</li>
+ * <li>if the destination of the edge has multiple inEdges, split ScheduleGroup by the edge.</li>
+ * <li>Otherwise, the source and the destination of the edge should be assigned same ScheduleGroup.</li>
+ * </ul>
+ * </li>
* </ul>
*/
@Annotates(ScheduleGroupProperty.class)
@@ -80,8 +80,9 @@ public final class DefaultScheduleGroupPass extends AnnotatingPass {
/**
* Constructor.
- * @param allowBroadcastWithinScheduleGroup whether to allow Broadcast edges within a ScheduleGroup or not
- * @param allowShuffleWithinScheduleGroup whether to allow Shuffle edges within a ScheduleGroup or not
+ *
+ * @param allowBroadcastWithinScheduleGroup whether to allow Broadcast edges within a ScheduleGroup or not
+ * @param allowShuffleWithinScheduleGroup whether to allow Shuffle edges within a ScheduleGroup or not
* @param allowMultipleInEdgesWithinScheduleGroup whether to allow vertices with multiple dependencies or not
*/
public DefaultScheduleGroupPass(final boolean allowBroadcastWithinScheduleGroup,
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DuplicateEdgeGroupSizePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DuplicateEdgeGroupSizePass.java
index 0e49943..6ebd2a2 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DuplicateEdgeGroupSizePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DuplicateEdgeGroupSizePass.java
@@ -42,27 +42,27 @@ public final class DuplicateEdgeGroupSizePass extends AnnotatingPass {
public IRDAG apply(final IRDAG dag) {
final HashMap<String, Integer> groupIdToGroupSize = new HashMap<>();
dag.topologicalDo(vertex -> dag.getIncomingEdgesOf(vertex)
- .forEach(e -> {
- final Optional<DuplicateEdgeGroupPropertyValue> duplicateEdgeGroupProperty =
- e.getPropertyValue(DuplicateEdgeGroupProperty.class);
- if (duplicateEdgeGroupProperty.isPresent()) {
- final String groupId = duplicateEdgeGroupProperty.get().getGroupId();
- final Integer currentCount = groupIdToGroupSize.getOrDefault(groupId, 0);
- groupIdToGroupSize.put(groupId, currentCount + 1);
- }
- }));
+ .forEach(e -> {
+ final Optional<DuplicateEdgeGroupPropertyValue> duplicateEdgeGroupProperty =
+ e.getPropertyValue(DuplicateEdgeGroupProperty.class);
+ if (duplicateEdgeGroupProperty.isPresent()) {
+ final String groupId = duplicateEdgeGroupProperty.get().getGroupId();
+ final Integer currentCount = groupIdToGroupSize.getOrDefault(groupId, 0);
+ groupIdToGroupSize.put(groupId, currentCount + 1);
+ }
+ }));
dag.topologicalDo(vertex -> dag.getIncomingEdgesOf(vertex)
- .forEach(e -> {
- final Optional<DuplicateEdgeGroupPropertyValue> duplicateEdgeGroupProperty =
- e.getPropertyValue(DuplicateEdgeGroupProperty.class);
- if (duplicateEdgeGroupProperty.isPresent()) {
- final String groupId = duplicateEdgeGroupProperty.get().getGroupId();
- if (groupIdToGroupSize.containsKey(groupId)) {
- duplicateEdgeGroupProperty.get().setGroupSize(groupIdToGroupSize.get(groupId));
- }
+ .forEach(e -> {
+ final Optional<DuplicateEdgeGroupPropertyValue> duplicateEdgeGroupProperty =
+ e.getPropertyValue(DuplicateEdgeGroupProperty.class);
+ if (duplicateEdgeGroupProperty.isPresent()) {
+ final String groupId = duplicateEdgeGroupProperty.get().getGroupId();
+ if (groupIdToGroupSize.containsKey(groupId)) {
+ duplicateEdgeGroupProperty.get().setGroupSize(groupIdToGroupSize.get(groupId));
}
- }));
+ }
+ }));
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/LargeShuffleAnnotatingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/LargeShuffleAnnotatingPass.java
index 6dc399b..120c598 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/LargeShuffleAnnotatingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/LargeShuffleAnnotatingPass.java
@@ -19,23 +19,26 @@
package org.apache.nemo.compiler.optimizer.pass.compiletime.annotating;
import org.apache.nemo.common.ir.IRDAG;
-import org.apache.nemo.common.ir.edge.executionproperty.*;
+import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.DataFlowProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.DataPersistenceProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.DataStoreProperty;
import org.apache.nemo.common.ir.vertex.executionproperty.ResourceSlotProperty;
import org.apache.nemo.common.ir.vertex.utility.StreamVertex;
import org.apache.nemo.compiler.optimizer.pass.compiletime.Requires;
/**
* This pass assumes that a StreamVertex was previously inserted to receive each shuffle edge.
- *
+ * <p>
* src - shuffle-edge - streamvertex - one-to-one-edge - dst
- *
+ * <p>
* (1) shuffle-edge
* Encode/compress into byte[], and have the receiver read data as the same byte[], rather than decompressing/decoding.
* Perform a push-based in-memory shuffle with discarding on.
- *
+ * <p>
* (2) streamvertex
* Ignore resource slots, such that all tasks fetch the in-memory input data blocks as soon as they become available.
- *
+ * <p>
* (3) one-to-one-edge
* Do not encode/compress the byte[]
* Perform a pull-based and on-disk data transfer with the DedicatedKeyPerElement.
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/PipeTransferForAllEdgesPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/PipeTransferForAllEdgesPass.java
index 37ad2cb..2436b35 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/PipeTransferForAllEdgesPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/PipeTransferForAllEdgesPass.java
@@ -37,8 +37,8 @@ public final class PipeTransferForAllEdgesPass extends AnnotatingPass {
public IRDAG apply(final IRDAG dag) {
dag.getVertices().forEach(vertex -> {
dag.getIncomingEdgesOf(vertex).stream()
- .forEach(edge -> edge.setPropertyPermanently(
- DataStoreProperty.of(DataStoreProperty.Value.Pipe)));
+ .forEach(edge -> edge.setPropertyPermanently(
+ DataStoreProperty.of(DataStoreProperty.Value.Pipe)));
});
return dag;
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceLocalityPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceLocalityPass.java
index 8294beb..b13e8ab 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceLocalityPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceLocalityPass.java
@@ -38,8 +38,8 @@ public final class ResourceLocalityPass extends AnnotatingPass {
public IRDAG apply(final IRDAG dag) {
// On every vertex, if ResourceLocalityProperty is not set, put it as true.
dag.getVertices().stream()
- .filter(v -> !v.getPropertyValue(ResourceLocalityProperty.class).isPresent())
- .forEach(v -> v.setProperty(ResourceLocalityProperty.of(true)));
+ .filter(v -> !v.getPropertyValue(ResourceLocalityProperty.class).isPresent())
+ .forEach(v -> v.setProperty(ResourceLocalityProperty.of(true)));
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSitePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSitePass.java
index 1820a7c..38d4e37 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSitePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSitePass.java
@@ -91,7 +91,7 @@ public final class ResourceSitePass extends AnnotatingPass {
}
/**
- * @param nodes to distribute the shares
+ * @param nodes to distribute the shares
* @param parallelism number of parallel tasks.
* @return share for each node.
*/
@@ -106,23 +106,23 @@ public final class ResourceSitePass extends AnnotatingPass {
}
/**
- * @param dag IR DAG.
+ * @param dag IR DAG.
* @param bandwidthSpecification bandwidth specification.
*/
private static void assignNodeShares(
- final IRDAG dag,
- final BandwidthSpecification bandwidthSpecification) {
+ final IRDAG dag,
+ final BandwidthSpecification bandwidthSpecification) {
dag.topologicalDo(irVertex -> {
final Collection<IREdge> inEdges = dag.getIncomingEdgesOf(irVertex);
final int parallelism = irVertex.getPropertyValue(ParallelismProperty.class)
- .orElseThrow(() -> new RuntimeException("Parallelism property required"));
+ .orElseThrow(() -> new RuntimeException("Parallelism property required"));
if (inEdges.size() == 0) {
// This vertex is root vertex.
// Fall back to setting even distribution
irVertex.setProperty(ResourceSiteProperty.of(EMPTY_MAP));
} else if (isOneToOneEdge(inEdges)) {
final Optional<HashMap<String, Integer>> property = inEdges.iterator().next().getSrc()
- .getPropertyValue(ResourceSiteProperty.class);
+ .getPropertyValue(ResourceSiteProperty.class);
irVertex.setProperty(ResourceSiteProperty.of(property.get()));
} else {
// This IRVertex has shuffle inEdge(s), or has multiple inEdges.
@@ -131,13 +131,13 @@ public final class ResourceSitePass extends AnnotatingPass {
final IRVertex parentVertex = edgeToIRVertex.getSrc();
final Map<String, Integer> parentShares = parentVertex.getPropertyValue(ResourceSiteProperty.class).get();
final int parentParallelism = parentVertex.getPropertyValue(ParallelismProperty.class)
- .orElseThrow(() -> new RuntimeException("Parallelism property required"));
+ .orElseThrow(() -> new RuntimeException("Parallelism property required"));
final Map<String, Integer> shares = parentShares.isEmpty() ? getEvenShares(bandwidthSpecification.getNodes(),
- parentParallelism) : parentShares;
+ parentParallelism) : parentShares;
for (final Map.Entry<String, Integer> element : shares.entrySet()) {
parentLocationShares.putIfAbsent(element.getKey(), 0);
parentLocationShares.put(element.getKey(),
- element.getValue() + parentLocationShares.get(element.getKey()));
+ element.getValue() + parentLocationShares.get(element.getKey()));
}
}
final double[] ratios = optimize(bandwidthSpecification, parentLocationShares);
@@ -164,14 +164,15 @@ public final class ResourceSitePass extends AnnotatingPass {
*/
private static boolean isOneToOneEdge(final Collection<IREdge> inEdges) {
return inEdges.size() == 1 && inEdges.iterator().next()
- .getPropertyValue(CommunicationPatternProperty.class).get()
- .equals(CommunicationPatternProperty.Value.OneToOne);
+ .getPropertyValue(CommunicationPatternProperty.class).get()
+ .equals(CommunicationPatternProperty.Value.OneToOne);
}
/**
* Computes share of parallelism that each node is responsible for.
+ *
* @param bandwidthSpecification provides bandwidth information between nodes
- * @param parentNodeShares shares of parallelism for the parent vertex
+ * @param parentNodeShares shares of parallelism for the parent vertex
* @return array of fractions of parallelism that each node is responsible for
*/
private static double[] optimize(final BandwidthSpecification bandwidthSpecification,
@@ -191,7 +192,7 @@ public final class ResourceSitePass extends AnnotatingPass {
uploadCoefficientVector[OBJECTIVE_COEFFICIENT_INDEX] = bandwidthSpecification.up(nodeName);
uploadCoefficientVector[nodeCoefficientIndex] = parentParallelismOnThisLocation;
constraints.add(new LinearConstraint(uploadCoefficientVector, Relationship.GEQ,
- parentParallelismOnThisLocation));
+ parentParallelismOnThisLocation));
// Download bandwidth
final double[] downloadCoefficientVector = new double[coefficientVectorSize];
@@ -226,7 +227,7 @@ public final class ResourceSitePass extends AnnotatingPass {
incrementor.setMaximalCount(2147483647);
LOG.info(String.format("Max iterations: %d", solver.getMaxIterations()));
final PointValuePair solved = solver.optimize(
- new LinearConstraintSet(constraints), objectiveFunction, GoalType.MINIMIZE);
+ new LinearConstraintSet(constraints), objectiveFunction, GoalType.MINIMIZE);
return Arrays.copyOfRange(solved.getPoint(), OBJECTIVE_COEFFICIENT_INDEX + 1, coefficientVectorSize);
} catch (final NoSuchFieldException | IllegalAccessException e) {
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSlotPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSlotPass.java
index abc4c77..25bac1e 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSlotPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ResourceSlotPass.java
@@ -38,8 +38,8 @@ public final class ResourceSlotPass extends AnnotatingPass {
public IRDAG apply(final IRDAG dag) {
// On every vertex, if ResourceSlotProperty is not set, put it as true.
dag.getVertices().stream()
- .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent())
- .forEach(v -> v.setProperty(ResourceSlotProperty.of(true)));
+ .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent())
+ .forEach(v -> v.setProperty(ResourceSlotProperty.of(true)));
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ShuffleEdgePushPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ShuffleEdgePushPass.java
index 007ce0e..9249a45 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ShuffleEdgePushPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/ShuffleEdgePushPass.java
@@ -47,7 +47,7 @@ public final class ShuffleEdgePushPass extends AnnotatingPass {
if (!inEdges.isEmpty()) {
inEdges.forEach(edge -> {
if (edge.getPropertyValue(CommunicationPatternProperty.class).get()
- .equals(CommunicationPatternProperty.Value.Shuffle)) {
+ .equals(CommunicationPatternProperty.Value.Shuffle)) {
edge.setProperty(DataFlowProperty.of(DataFlowProperty.Value.Push));
}
});
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourceDataStorePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourceDataStorePass.java
index 76136eb..2bc778e 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourceDataStorePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourceDataStorePass.java
@@ -62,25 +62,27 @@ public final class TransientResourceDataStorePass extends AnnotatingPass {
/**
* checks if the edge is from transient container to a reserved container.
+ *
* @param irEdge edge to check.
* @return whether or not the edge satisfies the condition.
*/
static boolean fromTransientToReserved(final IREdge irEdge) {
return ResourcePriorityProperty.TRANSIENT
- .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get())
- && ResourcePriorityProperty.RESERVED
- .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get());
+ .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get())
+ && ResourcePriorityProperty.RESERVED
+ .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get());
}
/**
* checks if the edge is from reserved container to a transient container.
+ *
* @param irEdge edge to check.
* @return whether or not the edge satisfies the condition.
*/
static boolean fromReservedToTransient(final IREdge irEdge) {
return ResourcePriorityProperty.RESERVED
- .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get())
- && ResourcePriorityProperty.TRANSIENT
- .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get());
+ .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get())
+ && ResourcePriorityProperty.TRANSIENT
+ .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get());
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourcePriorityPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourcePriorityPass.java
index 5988d61..46989de 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourcePriorityPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/TransientResourcePriorityPass.java
@@ -58,25 +58,27 @@ public final class TransientResourcePriorityPass extends AnnotatingPass {
/**
* Checks whether the irEdges have M2M relationship.
+ *
* @param irEdges irEdges to check.
* @return whether of not any of them has M2M relationship.
*/
private boolean hasM2M(final List<IREdge> irEdges) {
return irEdges.stream().anyMatch(edge ->
- edge.getPropertyValue(CommunicationPatternProperty.class).get()
- .equals(CommunicationPatternProperty.Value.Shuffle));
+ edge.getPropertyValue(CommunicationPatternProperty.class).get()
+ .equals(CommunicationPatternProperty.Value.Shuffle));
}
/**
* Checks whether the irEdges are all from reserved containers.
+ *
* @param irEdges irEdges to check.
* @return whether of not they are from reserved containers.
*/
private boolean allO2OFromReserved(final List<IREdge> irEdges) {
return irEdges.stream()
- .allMatch(edge -> CommunicationPatternProperty.Value.OneToOne.equals(
- edge.getPropertyValue(CommunicationPatternProperty.class).get())
- && edge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get().equals(
- ResourcePriorityProperty.RESERVED));
+ .allMatch(edge -> CommunicationPatternProperty.Value.OneToOne.equals(
+ edge.getPropertyValue(CommunicationPatternProperty.class).get())
+ && edge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get().equals(
+ ResourcePriorityProperty.RESERVED));
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/UpfrontCloningPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/UpfrontCloningPass.java
index 17492d4..7a2d9aa 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/UpfrontCloningPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/UpfrontCloningPass.java
@@ -39,17 +39,17 @@ public final class UpfrontCloningPass extends AnnotatingPass {
@Override
public IRDAG apply(final IRDAG dag) {
dag.getVertices().stream()
- .filter(vertex -> dag.getIncomingEdgesOf(vertex.getId())
- .stream()
- // TODO #198: Handle Un-cloneable Beam Sink Operators
- // only shuffle receivers (for now... as particular Beam sink operators fail when cloned)
- .anyMatch(edge ->
- edge.getPropertyValue(CommunicationPatternProperty.class)
- .orElseThrow(() -> new IllegalStateException())
- .equals(CommunicationPatternProperty.Value.Shuffle))
- )
- .forEach(vertex -> vertex.setProperty(
- ClonedSchedulingProperty.of(new ClonedSchedulingProperty.CloneConf()))); // clone upfront, always
+ .filter(vertex -> dag.getIncomingEdgesOf(vertex.getId())
+ .stream()
+ // TODO #198: Handle Un-cloneable Beam Sink Operators
+ // only shuffle receivers (for now... as particular Beam sink operators fail when cloned)
+ .anyMatch(edge ->
+ edge.getPropertyValue(CommunicationPatternProperty.class)
+ .orElseThrow(() -> new IllegalStateException())
+ .equals(CommunicationPatternProperty.Value.Shuffle))
+ )
+ .forEach(vertex -> vertex.setProperty(
+ ClonedSchedulingProperty.of(new ClonedSchedulingProperty.CloneConf()))); // clone upfront, always
return dag;
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/CompositePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/CompositePass.java
index 19c2a64..d27854f 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/CompositePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/CompositePass.java
@@ -37,6 +37,7 @@ public abstract class CompositePass extends CompileTimePass {
/**
* Constructor.
+ *
* @param passList list of compile time passes.
*/
public CompositePass(final List<CompileTimePass> passList) {
@@ -52,6 +53,7 @@ public abstract class CompositePass extends CompileTimePass {
/**
* Getter for list of compile time passes.
+ *
* @return the list of CompileTimePass.
*/
public final List<CompileTimePass> getPassList() {
@@ -65,7 +67,8 @@ public abstract class CompositePass extends CompileTimePass {
/**
* Recursively apply the give list of passes.
- * @param dag dag.
+ *
+ * @param dag dag.
* @param passIterator pass iterator.
* @return dag.
*/
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/DefaultCompositePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/DefaultCompositePass.java
index 3b869e0..49601f2 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/DefaultCompositePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/DefaultCompositePass.java
@@ -33,15 +33,15 @@ public final class DefaultCompositePass extends CompositePass {
*/
public DefaultCompositePass() {
super(Arrays.asList(
- new DefaultParallelismPass(),
- new DefaultEdgeEncoderPass(),
- new DefaultEdgeDecoderPass(),
- new DefaultDataStorePass(),
- new DefaultDataPersistencePass(),
- new DefaultScheduleGroupPass(),
- new CompressionPass(),
- new ResourceLocalityPass(),
- new ResourceSlotPass()
+ new DefaultParallelismPass(),
+ new DefaultEdgeEncoderPass(),
+ new DefaultEdgeDecoderPass(),
+ new DefaultDataStorePass(),
+ new DefaultDataPersistencePass(),
+ new DefaultScheduleGroupPass(),
+ new CompressionPass(),
+ new ResourceLocalityPass(),
+ new ResourceSlotPass()
));
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LargeShuffleCompositePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LargeShuffleCompositePass.java
index 2752c04..612fb0a 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LargeShuffleCompositePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LargeShuffleCompositePass.java
@@ -18,7 +18,7 @@
*/
package org.apache.nemo.compiler.optimizer.pass.compiletime.composite;
-import org.apache.nemo.compiler.optimizer.pass.compiletime.annotating.*;
+import org.apache.nemo.compiler.optimizer.pass.compiletime.annotating.LargeShuffleAnnotatingPass;
import org.apache.nemo.compiler.optimizer.pass.compiletime.reshaping.LargeShuffleReshapingPass;
import java.util.Arrays;
@@ -33,8 +33,8 @@ public final class LargeShuffleCompositePass extends CompositePass {
*/
public LargeShuffleCompositePass() {
super(Arrays.asList(
- new LargeShuffleReshapingPass(),
- new LargeShuffleAnnotatingPass()
+ new LargeShuffleReshapingPass(),
+ new LargeShuffleAnnotatingPass()
));
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LoopOptimizationCompositePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LoopOptimizationCompositePass.java
index e594f82..e36cbda 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LoopOptimizationCompositePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/LoopOptimizationCompositePass.java
@@ -34,11 +34,11 @@ public final class LoopOptimizationCompositePass extends CompositePass {
*/
public LoopOptimizationCompositePass() {
super(Arrays.asList(
- new LoopExtractionPass(),
- LoopOptimizations.getLoopFusionPass(),
- LoopOptimizations.getLoopInvariantCodeMotionPass(),
- new LoopUnrollingPass(), // Groups then unrolls loops.
- new DuplicateEdgeGroupSizePass()
+ new LoopExtractionPass(),
+ LoopOptimizations.getLoopFusionPass(),
+ LoopOptimizations.getLoopInvariantCodeMotionPass(),
+ new LoopUnrollingPass(), // Groups then unrolls loops.
+ new DuplicateEdgeGroupSizePass()
));
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/TransientResourceCompositePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/TransientResourceCompositePass.java
index bccffaf..a848488 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/TransientResourceCompositePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/composite/TransientResourceCompositePass.java
@@ -34,9 +34,9 @@ public final class TransientResourceCompositePass extends CompositePass {
*/
public TransientResourceCompositePass() {
super(Arrays.asList(
- new TransientResourcePriorityPass(),
- new TransientResourceDataStorePass(),
- new TransientResourceDataFlowPass()
+ new TransientResourcePriorityPass(),
+ new TransientResourceDataStorePass(),
+ new TransientResourceDataFlowPass()
));
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/CommonSubexpressionEliminationPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/CommonSubexpressionEliminationPass.java
index 9e066e1..fe2dff0 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/CommonSubexpressionEliminationPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/CommonSubexpressionEliminationPass.java
@@ -135,10 +135,11 @@ public final class CommonSubexpressionEliminationPass extends ReshapingPass {
/**
* merge equivalent operator vertices and add them to the provided builder.
- * @param ovs operator vertices that are to be merged (if there are no dependencies between them).
- * @param builder builder to add the merged vertices to.
- * @param dag dag to observe while adding them.
- * @param inEdges incoming edges information.
+ *
+ * @param ovs operator vertices that are to be merged (if there are no dependencies between them).
+ * @param builder builder to add the merged vertices to.
+ * @param dag dag to observe while adding them.
+ * @param inEdges incoming edges information.
* @param outEdges outgoing edges information.
*/
private static void mergeAndAddToBuilder(final List<OperatorVertex> ovs, final DAGBuilder<IRVertex, IREdge> builder,
@@ -161,7 +162,7 @@ public final class CommonSubexpressionEliminationPass extends ReshapingPass {
outEdges.getOrDefault(ov, new HashSet<>()).forEach(e -> {
outListToModify.remove(e);
final IREdge newIrEdge = new IREdge(e.getPropertyValue(CommunicationPatternProperty.class).get(),
- operatorVertexToUse, e.getDst());
+ operatorVertexToUse, e.getDst());
final Optional<EncoderFactory> encoderProperty = e.getPropertyValue(EncoderProperty.class);
if (encoderProperty.isPresent()) {
newIrEdge.setProperty(EncoderProperty.of(encoderProperty.get()));
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopExtractionPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopExtractionPass.java
index 1c37978..8c6b448 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopExtractionPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopExtractionPass.java
@@ -61,14 +61,15 @@ public final class LoopExtractionPass extends ReshapingPass {
/**
* This method finds the maximum loop vertex stack depth of a specific DAG. This is to handle nested loops.
+ *
* @param dag DAG to observe.
* @return The maximum stack depth of the DAG.
* @throws Exception exceptions through the way.
*/
private Integer findMaxLoopVertexStackDepth(final DAG<IRVertex, IREdge> dag) {
final OptionalInt maxDepth = dag.getVertices().stream().filter(dag::isCompositeVertex)
- .mapToInt(dag::getLoopStackDepthOf)
- .max();
+ .mapToInt(dag::getLoopStackDepthOf)
+ .max();
return maxDepth.orElse(0);
}
@@ -76,7 +77,8 @@ public final class LoopExtractionPass extends ReshapingPass {
* This part groups each iteration of loops together by observing the LoopVertex assigned to primitive operators,
* which is assigned by the NemoPipelineVisitor. This also shows in which depth of
* nested loops the function handles. It recursively calls itself from the maximum depth until 0.
- * @param dag DAG to process
+ *
+ * @param dag DAG to process
* @param depth the depth of the stack to process. Must be greater than 0.
* @return processed DAG.
* @throws Exception exceptions through the way.
@@ -143,9 +145,10 @@ public final class LoopExtractionPass extends ReshapingPass {
/**
* Method for connecting an element to a loop. That is, loop -> loop OR operator -> loop.
- * @param dag to observe the inEdges from.
- * @param builder to add the new edge to.
- * @param dstVertex the destination vertex that belongs to a certain loop.
+ *
+ * @param dag to observe the inEdges from.
+ * @param builder to add the new edge to.
+ * @param dstVertex the destination vertex that belongs to a certain loop.
* @param assignedLoopVertex the loop that dstVertex belongs to.
*/
private static void connectElementToLoop(final DAG<IRVertex, IREdge> dag, final DAGBuilder<IRVertex, IREdge> builder,
@@ -160,7 +163,7 @@ public final class LoopExtractionPass extends ReshapingPass {
} else { // loop -> loop connection
assignedLoopVertex.addDagIncomingEdge(irEdge);
final IREdge edgeToLoop = new IREdge(irEdge.getPropertyValue(CommunicationPatternProperty.class).get(),
- srcLoopVertex, assignedLoopVertex);
+ srcLoopVertex, assignedLoopVertex);
irEdge.copyExecutionPropertiesTo(edgeToLoop);
builder.connectVertices(edgeToLoop);
assignedLoopVertex.mapEdgeWithLoop(edgeToLoop, irEdge);
@@ -168,7 +171,7 @@ public final class LoopExtractionPass extends ReshapingPass {
} else { // operator -> loop
assignedLoopVertex.addDagIncomingEdge(irEdge);
final IREdge edgeToLoop = new IREdge(irEdge.getPropertyValue(CommunicationPatternProperty.class).get(),
- irEdge.getSrc(), assignedLoopVertex);
+ irEdge.getSrc(), assignedLoopVertex);
irEdge.copyExecutionPropertiesTo(edgeToLoop);
builder.connectVertices(edgeToLoop);
assignedLoopVertex.mapEdgeWithLoop(edgeToLoop, irEdge);
@@ -179,6 +182,7 @@ public final class LoopExtractionPass extends ReshapingPass {
/**
* This part rolls the repetitive LoopVertices into a single one, leaving only the root LoopVertex.
* Following iterations can be generated with the information included in the LoopVertex.
+ *
* @param dag DAG to process.
* @return Processed DAG.
* @throws Exception exceptions through the way.
@@ -222,7 +226,7 @@ public final class LoopExtractionPass extends ReshapingPass {
// Zip current vertices together. We rely on the fact that getTopologicalSort() brings consistent results.
final Iterator<IRVertex> verticesOfRootLoopVertex =
- finalRootLoopVertex.getDAG().getTopologicalSort().iterator();
+ finalRootLoopVertex.getDAG().getTopologicalSort().iterator();
final Iterator<IRVertex> verticesOfCurrentLoopVertex = loopVertex.getDAG().getTopologicalSort().iterator();
// Map of (RolledVertex --> (Root)Vertex)
final HashMap<IRVertex, IRVertex> equivalentVertices = equivalentVerticesOfLoops.get(finalRootLoopVertex);
@@ -248,13 +252,13 @@ public final class LoopExtractionPass extends ReshapingPass {
// add the new IREdge to the iterative incoming edges list.
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- equivalentSrcVertex, equivalentDstVertex);
+ equivalentSrcVertex, equivalentDstVertex);
edge.copyExecutionPropertiesTo(newIrEdge);
finalRootLoopVertex.addIterativeIncomingEdge(newIrEdge);
} else {
// src is from outside the previous loop. vertex outside previous loop -> DAG.
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- srcVertex, equivalentDstVertex);
+ srcVertex, equivalentDstVertex);
edge.copyExecutionPropertiesTo(newIrEdge);
finalRootLoopVertex.addNonIterativeIncomingEdge(newIrEdge);
}
@@ -267,7 +271,7 @@ public final class LoopExtractionPass extends ReshapingPass {
final IRVertex equivalentSrcVertex = equivalentVertices.get(srcVertex);
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- equivalentSrcVertex, dstVertex);
+ equivalentSrcVertex, dstVertex);
edge.copyExecutionPropertiesTo(newIrEdge);
finalRootLoopVertex.addDagOutgoingEdge(newIrEdge);
finalRootLoopVertex.mapEdgeWithLoop(loopVertex.getEdgeWithLoop(edge), newIrEdge);
@@ -284,9 +288,10 @@ public final class LoopExtractionPass extends ReshapingPass {
/**
* Adds the vertex and the incoming edges of the vertex to the builder.
- * @param builder Builder that it adds to.
- * @param irVertex Vertex to add.
- * @param dag DAG to observe the incoming edges of the vertex.
+ *
+ * @param builder Builder that it adds to.
+ * @param irVertex Vertex to add.
+ * @param dag DAG to observe the incoming edges of the vertex.
* @param loopVerticesOfSameLoop List that keeps track of the iterations of the identical loop.
*/
private static void addVertexToBuilder(final DAGBuilder<IRVertex, IREdge> builder, final DAG<IRVertex, IREdge> dag,
@@ -312,7 +317,7 @@ public final class LoopExtractionPass extends ReshapingPass {
builder.connectVertices(edge);
} else {
final IREdge newIrEdge = new IREdge(edge.getPropertyValue(CommunicationPatternProperty.class).get(),
- firstEquivalentVertex, irVertex);
+ firstEquivalentVertex, irVertex);
edge.copyExecutionPropertiesTo(newIrEdge);
builder.connectVertices(newIrEdge);
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java
index 56013c0..8ff03bf 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java
@@ -59,11 +59,12 @@ public final class LoopOptimizations {
/**
* Static function to collect LoopVertices.
- * @param dag DAG to observe.
+ *
+ * @param dag DAG to observe.
* @param loopVertices Map to save the LoopVertices to, according to their termination conditions.
- * @param inEdges incoming edges of LoopVertices.
- * @param outEdges outgoing Edges of LoopVertices.
- * @param builder builder to build the rest of the DAG on.
+ * @param inEdges incoming edges of LoopVertices.
+ * @param outEdges outgoing Edges of LoopVertices.
+ * @param builder builder to build the rest of the DAG on.
*/
private static void collectLoopVertices(final DAG<IRVertex, IREdge> dag,
final List<LoopVertex> loopVertices,
@@ -209,12 +210,13 @@ public final class LoopOptimizations {
/**
* Merge the list of loopVertices into a single LoopVertex.
+ *
* @param loopVertices list of LoopVertices to merge.
* @return the merged single LoopVertex.
*/
private LoopVertex mergeLoopVertices(final Set<LoopVertex> loopVertices) {
final String newName =
- String.join("+", loopVertices.stream().map(LoopVertex::getName).collect(Collectors.toList()));
+ String.join("+", loopVertices.stream().map(LoopVertex::getName).collect(Collectors.toList()));
final LoopVertex mergedLoopVertex = new LoopVertex(newName);
loopVertices.forEach(loopVertex -> {
final DAG<IRVertex, IREdge> dagToCopy = loopVertex.getDAG();
@@ -224,9 +226,9 @@ public final class LoopOptimizations {
});
loopVertex.getDagIncomingEdges().forEach((v, es) -> es.forEach(mergedLoopVertex::addDagIncomingEdge));
loopVertex.getIterativeIncomingEdges().forEach((v, es) ->
- es.forEach(mergedLoopVertex::addIterativeIncomingEdge));
+ es.forEach(mergedLoopVertex::addIterativeIncomingEdge));
loopVertex.getNonIterativeIncomingEdges().forEach((v, es) ->
- es.forEach(mergedLoopVertex::addNonIterativeIncomingEdge));
+ es.forEach(mergedLoopVertex::addNonIterativeIncomingEdge));
loopVertex.getDagOutgoingEdges().forEach((v, es) -> es.forEach(mergedLoopVertex::addDagOutgoingEdge));
});
return mergedLoopVertex;
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopUnrollingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopUnrollingPass.java
index 3d8b119..46a23ab 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopUnrollingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopUnrollingPass.java
@@ -48,6 +48,7 @@ public final class LoopUnrollingPass extends ReshapingPass {
/**
* A function that recursively calls Unroll until there are no more LoopVertex left.
+ *
* @param dag DAG to process.
* @return DAG without LoopVertex.
*/
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/ReshapingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/ReshapingPass.java
index dd80faa..e56d46d 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/ReshapingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/ReshapingPass.java
@@ -35,12 +35,13 @@ public abstract class ReshapingPass extends CompileTimePass {
/**
* Constructor.
+ *
* @param cls the reshaping pass class.
*/
public ReshapingPass(final Class<? extends ReshapingPass> cls) {
final Requires requires = cls.getAnnotation(Requires.class);
this.prerequisiteExecutionProperties = requires == null
- ? new HashSet<>() : new HashSet<>(Arrays.asList(requires.value()));
+ ? new HashSet<>() : new HashSet<>(Arrays.asList(requires.value()));
}
/**
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingSkewReshapingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingSkewReshapingPass.java
index 8db8085..efb9416 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingSkewReshapingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingSkewReshapingPass.java
@@ -22,7 +22,9 @@ import org.apache.nemo.common.KeyExtractor;
import org.apache.nemo.common.dag.Edge;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.common.ir.edge.IREdge;
-import org.apache.nemo.common.ir.edge.executionproperty.*;
+import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.DataStoreProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.KeyExtractorProperty;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.ir.vertex.utility.MessageAggregatorVertex;
import org.apache.nemo.common.ir.vertex.utility.MessageBarrierVertex;
@@ -31,25 +33,28 @@ import org.apache.nemo.compiler.optimizer.pass.compiletime.Requires;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
import java.util.stream.Collectors;
/**
* Optimizes the PartitionSet property of shuffle edges to handle data skews using the SamplingVertex.
- *
+ * <p>
* This pass effectively partitions the IRDAG by non-oneToOne edges, clones each subDAG partition using SamplingVertex
* to process sampled data, and executes each cloned partition prior to executing the corresponding original partition.
- *
+ * <p>
* Suppose the IRDAG is partitioned into three sub-DAG partitions with shuffle dependencies as follows:
* P1 - P2 - P3
- *
+ * <p>
* Then, this pass will produce something like:
* P1' - P1
- * - P2' - P2 - P3
+ * - P2' - P2 - P3
* where Px' consists of SamplingVertex objects that clone the execution of Px.
* (P3 is not cloned here because it is a sink partition, and none of the outgoing edges of its vertices needs to be
* optimized)
- *
+ * <p>
* For each Px' this pass also inserts a MessageBarrierVertex, to use its data statistics for dynamically optimizing
* the execution behaviors of Px.
*/
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewHandlingUtil.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewHandlingUtil.java
index 6231ba4..c176ad6 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewHandlingUtil.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewHandlingUtil.java
@@ -20,7 +20,10 @@ package org.apache.nemo.compiler.optimizer.pass.compiletime.reshaping;
import org.apache.nemo.common.KeyExtractor;
import org.apache.nemo.common.Pair;
-import org.apache.nemo.common.coder.*;
+import org.apache.nemo.common.coder.LongDecoderFactory;
+import org.apache.nemo.common.coder.LongEncoderFactory;
+import org.apache.nemo.common.coder.PairDecoderFactory;
+import org.apache.nemo.common.coder.PairEncoderFactory;
import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.DecoderProperty;
import org.apache.nemo.common.ir.edge.executionproperty.EncoderProperty;
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewReshapingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewReshapingPass.java
index 046dd75..0b843dd 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewReshapingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SkewReshapingPass.java
@@ -21,15 +21,18 @@ package org.apache.nemo.compiler.optimizer.pass.compiletime.reshaping;
import org.apache.nemo.common.KeyExtractor;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.common.ir.edge.IREdge;
-import org.apache.nemo.common.ir.edge.executionproperty.*;
+import org.apache.nemo.common.ir.edge.executionproperty.AdditionalOutputTagProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
+import org.apache.nemo.common.ir.edge.executionproperty.KeyExtractorProperty;
import org.apache.nemo.common.ir.vertex.utility.MessageAggregatorVertex;
import org.apache.nemo.common.ir.vertex.utility.MessageBarrierVertex;
-import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
import org.apache.nemo.compiler.optimizer.pass.compiletime.Requires;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -37,7 +40,7 @@ import java.util.stream.Collectors;
* Pass to reshape the IR DAG for skew handling.
* We insert a {@link MessageBarrierVertex} for each shuffle edge,
* and aggregate messages for multiple same-destination shuffle edges.
- * */
+ */
@Requires(CommunicationPatternProperty.class)
public final class SkewReshapingPass extends ReshapingPass {
private static final Logger LOG = LoggerFactory.getLogger(SkewReshapingPass.class.getName());
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/RunTimePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/RunTimePass.java
index cfdf936..81c023c 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/RunTimePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/RunTimePass.java
@@ -25,6 +25,7 @@ import java.util.function.BiFunction;
/**
* Abstract class for dynamic optimization passes, for dynamically optimizing the IRDAG.
+ *
* @param <T> type of the message used for dynamic optimization.
*/
public abstract class RunTimePass<T> extends Pass implements BiFunction<IRDAG, Message<T>, IRDAG> {
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/SkewRunTimePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/SkewRunTimePass.java
index 5dcfac6..54e4599 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/SkewRunTimePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/runtime/SkewRunTimePass.java
@@ -19,9 +19,9 @@
package org.apache.nemo.compiler.optimizer.pass.runtime;
import org.apache.nemo.common.HashRange;
+import org.apache.nemo.common.KeyRange;
import org.apache.nemo.common.Pair;
import org.apache.nemo.common.ir.IRDAG;
-import org.apache.nemo.common.KeyRange;
import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.PartitionSetProperty;
import org.apache.nemo.common.ir.edge.executionproperty.PartitionerProperty;
@@ -102,10 +102,10 @@ public final class SkewRunTimePass extends RunTimePass<Map<Object, Long>> {
* redistribute the key range of partitions with approximate size of (total size of partitions / the number of tasks).
* Assumption: the returned key of the partitioner is always 0 or positive integer.
*
- * @param keyToCountMap statistics.
- * @param partitioner used.
+ * @param keyToCountMap statistics.
+ * @param partitioner used.
* @param numOfPartitions created.
- * @param dstParallelism of the destination vertex.
+ * @param dstParallelism of the destination vertex.
* @return an optimal PartitionSetProperty and a ResourceAntiAffinityProperty.
*/
Pair<PartitionSetProperty, ResourceAntiAffinityProperty> analyzeMessage(final Map<Object, Long> keyToCountMap,
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/BasicPullPolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/BasicPullPolicy.java
index c304a1e..d9f063d 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/BasicPullPolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/BasicPullPolicy.java
@@ -29,9 +29,9 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
*/
public final class BasicPullPolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new AggressiveSpeculativeCloningPass())
- .registerCompileTimePass(new DefaultScheduleGroupPass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new AggressiveSpeculativeCloningPass())
+ .registerCompileTimePass(new DefaultScheduleGroupPass());
private final Policy policy;
/**
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/ConditionalLargeShufflePolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/ConditionalLargeShufflePolicy.java
index af89f89..fac310f 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/ConditionalLargeShufflePolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/ConditionalLargeShufflePolicy.java
@@ -30,10 +30,10 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
*/
public final class ConditionalLargeShufflePolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new LargeShuffleCompositePass(), dag -> getMaxParallelism(dag) > 300)
- .registerCompileTimePass(new LoopOptimizationCompositePass())
- .registerCompileTimePass(new DefaultCompositePass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new LargeShuffleCompositePass(), dag -> getMaxParallelism(dag) > 300)
+ .registerCompileTimePass(new LoopOptimizationCompositePass())
+ .registerCompileTimePass(new DefaultCompositePass());
private final Policy policy;
/**
@@ -45,13 +45,14 @@ public final class ConditionalLargeShufflePolicy implements Policy {
/**
* Returns the maximum parallelism of the vertices of a IR DAG.
+ *
* @param dag dag to observe.
* @return the maximum parallelism, or 1 by default.
*/
private static int getMaxParallelism(final IRDAG dag) {
return dag.getVertices().stream()
- .mapToInt(vertex -> vertex.getPropertyValue(ParallelismProperty.class).orElse(1))
- .max().orElse(1);
+ .mapToInt(vertex -> vertex.getPropertyValue(ParallelismProperty.class).orElse(1))
+ .max().orElse(1);
}
@Override
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DataSkewPolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DataSkewPolicy.java
index b25ca70..a8ab38e 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DataSkewPolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DataSkewPolicy.java
@@ -31,11 +31,11 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.SkewRunTimePass;
*/
public final class DataSkewPolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new DefaultParallelismPass()) // SkewCompositePass relies on parallelism.
- .registerRunTimePass(new SkewRunTimePass(), new SkewCompositePass())
- .registerCompileTimePass(new LoopOptimizationCompositePass())
- .registerCompileTimePass(new DefaultCompositePass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new DefaultParallelismPass()) // SkewCompositePass relies on parallelism.
+ .registerRunTimePass(new SkewRunTimePass(), new SkewCompositePass())
+ .registerCompileTimePass(new LoopOptimizationCompositePass())
+ .registerCompileTimePass(new DefaultCompositePass());
private final Policy policy;
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicy.java
index c1b626e..b4136b4 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicy.java
@@ -27,8 +27,8 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
*/
public final class DefaultPolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new DefaultCompositePass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new DefaultCompositePass());
private final Policy policy;
/**
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicyWithSeparatePass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicyWithSeparatePass.java
index 8911fab..df7eaf6 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicyWithSeparatePass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DefaultPolicyWithSeparatePass.java
@@ -34,9 +34,9 @@ import java.util.Arrays;
*/
public final class DefaultPolicyWithSeparatePass implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new DefaultParallelismPass())
- .registerCompileTimePass(new RefactoredPass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new DefaultParallelismPass())
+ .registerCompileTimePass(new RefactoredPass());
private final Policy policy;
/**
@@ -65,8 +65,8 @@ public final class DefaultPolicyWithSeparatePass implements Policy {
*/
RefactoredPass() {
super(Arrays.asList(
- new DefaultDataStorePass(),
- new DefaultScheduleGroupPass()
+ new DefaultDataStorePass(),
+ new DefaultScheduleGroupPass()
));
}
}
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DisaggregationPolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DisaggregationPolicy.java
index 66846c2..2cc4b8a 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DisaggregationPolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/DisaggregationPolicy.java
@@ -29,10 +29,10 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
*/
public final class DisaggregationPolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new DisaggregationEdgeDataStorePass())
- .registerCompileTimePass(new LoopOptimizationCompositePass())
- .registerCompileTimePass(new DefaultCompositePass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new DisaggregationEdgeDataStorePass())
+ .registerCompileTimePass(new LoopOptimizationCompositePass())
+ .registerCompileTimePass(new DefaultCompositePass());
private final Policy policy;
/**
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/LargeShufflePolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/LargeShufflePolicy.java
index 2644106..5b8e3f1 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/LargeShufflePolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/LargeShufflePolicy.java
@@ -29,10 +29,10 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
*/
public final class LargeShufflePolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new LargeShuffleCompositePass())
- .registerCompileTimePass(new LoopOptimizationCompositePass())
- .registerCompileTimePass(new DefaultCompositePass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new LargeShuffleCompositePass())
+ .registerCompileTimePass(new LoopOptimizationCompositePass())
+ .registerCompileTimePass(new DefaultCompositePass());
private final Policy policy;
/**
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/Policy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/Policy.java
index 71b78dd..9d4110d 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/Policy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/Policy.java
@@ -41,7 +41,7 @@ public interface Policy extends Serializable {
/**
* Optimize the DAG with the run-time optimizations.
*
- * @param dag input DAG.
+ * @param dag input DAG.
* @param message from the DAG execution.
*/
IRDAG runRunTimeOptimizations(final IRDAG dag, final Message<?> message);
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilder.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilder.java
index 69abb6a..22142e2 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilder.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilder.java
@@ -65,6 +65,7 @@ public final class PolicyBuilder {
/**
* Register a compile time pass.
+ *
* @param compileTimePass the compile time pass to register.
* @return the PolicyBuilder which registers the compileTimePass.
*/
@@ -79,7 +80,7 @@ public final class PolicyBuilder {
// Check prerequisite execution properties.
if (!annotatedExecutionProperties.containsAll(compileTimePass.getPrerequisiteExecutionProperties())) {
throw new CompileTimeOptimizationException("Prerequisite ExecutionProperty hasn't been met for "
- + compileTimePass.getClass().getSimpleName());
+ + compileTimePass.getClass().getSimpleName());
}
// check annotation of annotating passes.
@@ -95,8 +96,9 @@ public final class PolicyBuilder {
/**
* Register compile time pass with its condition under which to run the pass.
+ *
* @param compileTimePass the compile time pass to register.
- * @param condition condition under which to run the pass.
+ * @param condition condition under which to run the pass.
* @return the PolicyBuilder which registers the compileTimePass.
*/
public PolicyBuilder registerCompileTimePass(final CompileTimePass compileTimePass,
@@ -107,7 +109,8 @@ public final class PolicyBuilder {
/**
* Register a run time pass.
- * @param runTimePass the runtime pass to register.
+ *
+ * @param runTimePass the runtime pass to register.
* @param runTimePassRegisterer the compile time pass that triggers the runtime pass.
* @return the PolicyBuilder which registers the runTimePass and the runTimePassRegisterer.
*/
@@ -120,9 +123,10 @@ public final class PolicyBuilder {
/**
* Register a run time pass.
- * @param runTimePass the runtime pass to register.
+ *
+ * @param runTimePass the runtime pass to register.
* @param runTimePassRegisterer the compile time pass that triggers the runtime pass.
- * @param condition condition under which to run the pass.
+ * @param condition condition under which to run the pass.
* @return the PolicyBuilder which registers the runTimePass and the runTimePassRegisterer.
*/
public PolicyBuilder registerRunTimePass(final RunTimePass<?> runTimePass,
@@ -134,6 +138,7 @@ public final class PolicyBuilder {
/**
* Getter for compile time passes.
+ *
* @return the list of compile time passes.
*/
public List<CompileTimePass> getCompileTimePasses() {
@@ -142,6 +147,7 @@ public final class PolicyBuilder {
/**
* Getter for run time passes.
+ *
* @return the list of run time passes.
*/
public Set<RunTimePass<?>> getRunTimePasses() {
@@ -150,6 +156,7 @@ public final class PolicyBuilder {
/**
* Build a policy using compileTimePasses and runTimePasses in this object.
+ *
* @return the built Policy.
*/
public Policy build() {
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyImpl.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyImpl.java
index 956e00b..0e8bcb2 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyImpl.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/PolicyImpl.java
@@ -45,8 +45,9 @@ public final class PolicyImpl implements Policy {
/**
* Constructor.
+ *
* @param compileTimePasses compile time passes of the policy.
- * @param runTimePasses run time passes of the policy.
+ * @param runTimePasses run time passes of the policy.
*/
public PolicyImpl(final List<CompileTimePass> compileTimePasses, final Set<RunTimePass<?>> runTimePasses) {
this.compileTimePasses = compileTimePasses;
@@ -61,8 +62,9 @@ public final class PolicyImpl implements Policy {
/**
* A recursive method to process each pass one-by-one to the given DAG.
- * @param dag DAG to process.
- * @param passes passes to apply.
+ *
+ * @param dag DAG to process.
+ * @param passes passes to apply.
* @param dagDirectory directory to save the DAG information.
* @return the processed DAG.
* @throws Exception Exceptions on the way.
@@ -116,8 +118,9 @@ public final class PolicyImpl implements Policy {
/**
* Checks if the annotating pass hasn't modified the DAG structure.
* It checks if the number of Vertices and Edges are the same.
+ *
* @param before DAG before modification.
- * @param after DAG after modification.
+ * @param after DAG after modification.
* @return true if there is no problem, false if there is a problem.
*/
private static Boolean checkAnnotatingPass(final IRDAG before, final IRDAG after) {
@@ -159,8 +162,9 @@ public final class PolicyImpl implements Policy {
/**
* Checks if the reshaping pass hasn't modified execution properties.
* It checks if all of its vertices and edges have the same execution properties as before (if it existed then).
+ *
* @param before DAG before modification.
- * @param after DAG after modification.
+ * @param after DAG after modification.
* @return true if there is no problem, false if there is a problem.
*/
private static Boolean checkReshapingPass(final IRDAG before, final IRDAG after) {
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/SamplingLargeShuffleSkewPolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/SamplingLargeShuffleSkewPolicy.java
index 84d9470..6eac660 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/SamplingLargeShuffleSkewPolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/SamplingLargeShuffleSkewPolicy.java
@@ -20,7 +20,9 @@ package org.apache.nemo.compiler.optimizer.policy;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.compiler.optimizer.pass.compiletime.annotating.DefaultParallelismPass;
-import org.apache.nemo.compiler.optimizer.pass.compiletime.composite.*;
+import org.apache.nemo.compiler.optimizer.pass.compiletime.composite.DefaultCompositePass;
+import org.apache.nemo.compiler.optimizer.pass.compiletime.composite.LargeShuffleCompositePass;
+import org.apache.nemo.compiler.optimizer.pass.compiletime.composite.LoopOptimizationCompositePass;
import org.apache.nemo.compiler.optimizer.pass.compiletime.reshaping.SamplingSkewReshapingPass;
import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
import org.apache.nemo.compiler.optimizer.pass.runtime.SkewRunTimePass;
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/TransientResourcePolicy.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/TransientResourcePolicy.java
index e6d6a59..e059010 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/TransientResourcePolicy.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/policy/TransientResourcePolicy.java
@@ -29,10 +29,10 @@ import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
*/
public final class TransientResourcePolicy implements Policy {
public static final PolicyBuilder BUILDER =
- new PolicyBuilder()
- .registerCompileTimePass(new TransientResourceCompositePass())
- .registerCompileTimePass(new LoopOptimizationCompositePass())
- .registerCompileTimePass(new DefaultCompositePass());
+ new PolicyBuilder()
+ .registerCompileTimePass(new TransientResourceCompositePass())
+ .registerCompileTimePass(new LoopOptimizationCompositePass())
+ .registerCompileTimePass(new DefaultCompositePass());
private final Policy policy;
/**
diff --git a/compiler/optimizer/src/test/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilderTest.java b/compiler/optimizer/src/test/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilderTest.java
index bfb2654..1e1721a 100644
--- a/compiler/optimizer/src/test/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilderTest.java
+++ b/compiler/optimizer/src/test/java/org/apache/nemo/compiler/optimizer/policy/PolicyBuilderTest.java
@@ -49,9 +49,9 @@ public final class PolicyBuilderTest {
public void testShouldFailPolicy() {
try {
final Policy failPolicy = new PolicyBuilder()
- .registerCompileTimePass(new TransientResourceCompositePass())
- .registerCompileTimePass(new DefaultScheduleGroupPass())
- .build();
+ .registerCompileTimePass(new TransientResourceCompositePass())
+ .registerCompileTimePass(new DefaultScheduleGroupPass())
+ .build();
} catch (Exception e) { // throw an exception if default execution properties are not set.
assertTrue(e instanceof CompileTimeOptimizationException);
assertTrue(e.getMessage().contains("Prerequisite ExecutionProperty hasn't been met"));
diff --git a/compiler/pom.xml b/compiler/pom.xml
index bdd7fce..ad85ef5 100644
--- a/compiler/pom.xml
+++ b/compiler/pom.xml
@@ -17,7 +17,8 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nemo-project</artifactId>
diff --git a/compiler/test/pom.xml b/compiler/test/pom.xml
index c12027f..69f3747 100644
--- a/compiler/test/pom.xml
+++ b/compiler/test/pom.xml
@@ -17,71 +17,72 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler</artifactId>
- <version>0.2-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
- <artifactId>nemo-compiler-test</artifactId>
- <name>Nemo Compiler Test</name>
+ <artifactId>nemo-compiler-test</artifactId>
+ <name>Nemo Compiler Test</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-conf</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-client</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-compiler-frontend-beam</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.nemo</groupId>
- <artifactId>nemo-examples-beam</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${mockito.version}</version>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>${powermock.version}</version>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito2</artifactId>
- <version>${powermock.version}</version>
- </dependency>
- <dependency>
- <!--
- This is needed to view the logs when running unit tests.
- See https://dzone.com/articles/how-configure-slf4j-different for details.
- -->
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>1.6.2</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-conf</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-compiler-frontend-beam</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nemo</groupId>
+ <artifactId>nemo-examples-beam</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock.version}</version>
+ </dependency>
+ <dependency>
+ <!--
+ This is needed to view the logs when running unit tests.
+ See https://dzone.com/articles/how-configure-slf4j-different for details.
+ -->
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.6.2</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git a/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java b/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java
index 1ed114a..3f5001a 100644
--- a/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java
+++ b/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java
@@ -18,10 +18,10 @@
*/
package org.apache.nemo.compiler;
+import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.common.test.ArgBuilder;
import org.apache.nemo.conf.JobConf;
-import org.apache.nemo.client.JobLauncher;
import org.apache.reef.tang.Configuration;
import org.apache.reef.tang.Injector;
import org.apache.reef.tang.Tang;
@@ -45,6 +45,7 @@ public final class CompilerTestUtil {
/**
* Find the root directory of Nemo project, ascending directory hierarchy one by one starting from {@code curDir}.
* This method distinguishes the root with "LICENSE" file.
+ *
* @param curDir the current directory
* @return the absolute path of the root directory
*/
@@ -77,8 +78,8 @@ public final class CompilerTestUtil {
userMainMethodArgs = injector.getNamedInstance(JobConf.UserMainArguments.class).split(" ");
} catch (final Exception e) {
throw new RuntimeException("An exception occurred while processing configuration for invoking user main. "
- + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking "
- + "on JobLauncher and because of this Tang may misbehave afterwards.", e);
+ + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking "
+ + "on JobLauncher and because of this Tang may misbehave afterwards.", e);
}
final Class userMainClass = Class.forName(userMainClassName);
final Method userMainMethod = userMainClass.getMethod("main", String[].class);
@@ -97,9 +98,9 @@ public final class CompilerTestUtil {
final String main = "org.apache.nemo.examples.beam.WordCount";
final ArgBuilder mrArgBuilder = new ArgBuilder()
- .addJobId("WordCount")
- .addUserMain(main)
- .addUserArgs(input, output);
+ .addJobId("WordCount")
+ .addUserMain(main)
+ .addUserArgs(input, output);
return compileDAG(mrArgBuilder.build());
}
@@ -110,9 +111,9 @@ public final class CompilerTestUtil {
final String main = "org.apache.nemo.examples.beam.AlternatingLeastSquare";
final ArgBuilder alsArgBuilder = new ArgBuilder()
- .addJobId("AlternatingLeastSquare")
- .addUserMain(main)
- .addUserArgs(input, numFeatures, numIteration);
+ .addJobId("AlternatingLeastSquare")
+ .addUserMain(main)
+ .addUserArgs(input, numFeatures, numIteration);
return compileDAG(alsArgBuilder.build());
}
@@ -123,9 +124,9 @@ public final class CompilerTestUtil {
final String main = "org.apache.nemo.examples.beam.AlternatingLeastSquareInefficient";
final ArgBuilder alsArgBuilder = new ArgBuilder()
- .addJobId("AlternatingLeastSquareInefficient")
- .addUserMain(main)
- .addUserArgs(input, numFeatures, numIteration);
+ .addJobId("AlternatingLeastSquareInefficient")
+ .addUserMain(main)
+ .addUserArgs(input, numFeatures, numIteration);
return compileDAG(alsArgBuilder.build());
}
@@ -137,9 +138,9 @@ public final class CompilerTestUtil {
final String main = "org.apache.nemo.examples.beam.MultinomialLogisticRegression";
final ArgBuilder mlrArgBuilder = new ArgBuilder()
- .addJobId("MultinomialLogisticRegression")
- .addUserMain(main)
- .addUserArgs(input, numFeatures, numClasses, numIteration);
+ .addJobId("MultinomialLogisticRegression")
+ .addUserMain(main)
+ .addUserArgs(input, numFeatures, numClasses, numIteration);
return compileDAG(mlrArgBuilder.build());
}
}
diff --git a/compiler/test/src/main/java/org/apache/nemo/compiler/optimizer/policy/TestPolicy.java b/compiler/test/src/main/java/org/apache/nemo/compiler/optimizer/policy/TestPolicy.java
index 6b5b8c5..ec03325 100644
--- a/compiler/test/src/main/java/org/apache/nemo/compiler/optimizer/policy/TestPolicy.java
+++ b/compiler/test/src/main/java/org/apache/nemo/compiler/optimizer/policy/TestPolicy.java
@@ -20,10 +20,13 @@ package org.apache.nemo.compiler.optimizer.policy;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.compiler.optimizer.pass.compiletime.CompileTimePass;
-import org.apache.nemo.compiler.optimizer.pass.compiletime.annotating.*;
+import org.apache.nemo.compiler.optimizer.pass.compiletime.annotating.DefaultScheduleGroupPass;
+import org.apache.nemo.compiler.optimizer.pass.compiletime.annotating.ShuffleEdgePushPass;
import org.apache.nemo.compiler.optimizer.pass.runtime.Message;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
/**
* A policy for tests.
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/backend/nemo/DAGConverterTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/backend/nemo/DAGConverterTest.java
index 5a25c79..58367fb 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/backend/nemo/DAGConverterTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/backend/nemo/DAGConverterTest.java
@@ -25,18 +25,18 @@ import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.CommunicationPatternProperty;
import org.apache.nemo.common.ir.edge.executionproperty.DataFlowProperty;
import org.apache.nemo.common.ir.edge.executionproperty.DataStoreProperty;
-import org.apache.nemo.common.ir.vertex.SourceVertex;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.ir.vertex.OperatorVertex;
+import org.apache.nemo.common.ir.vertex.SourceVertex;
import org.apache.nemo.common.ir.vertex.executionproperty.ParallelismProperty;
import org.apache.nemo.common.ir.vertex.executionproperty.ResourcePriorityProperty;
import org.apache.nemo.common.ir.vertex.transform.Transform;
import org.apache.nemo.common.test.EmptyComponents;
+import org.apache.nemo.compiler.optimizer.policy.TestPolicy;
import org.apache.nemo.conf.JobConf;
import org.apache.nemo.runtime.common.plan.PhysicalPlanGenerator;
import org.apache.nemo.runtime.common.plan.Stage;
import org.apache.nemo.runtime.common.plan.StageEdge;
-import org.apache.nemo.compiler.optimizer.policy.TestPolicy;
import org.apache.reef.tang.Injector;
import org.apache.reef.tang.Tang;
import org.junit.Before;
@@ -77,7 +77,7 @@ public final class DAGConverterTest {
irDAGBuilder.connectVertices(e);
final IRDAG irDAG = new TestPolicy().runCompileTimeOptimization(
- new IRDAG(irDAGBuilder.buildWithoutSourceSinkCheck()), DAG.EMPTY_DAG_DIRECTORY);
+ new IRDAG(irDAGBuilder.buildWithoutSourceSinkCheck()), DAG.EMPTY_DAG_DIRECTORY);
final DAG<Stage, StageEdge> DAGOfStages = physicalPlanGenerator.stagePartitionIrDAG(irDAG);
final DAG<Stage, StageEdge> physicalDAG = physicalPlanGenerator.apply(irDAG);
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendALSTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendALSTest.java
index 5be2bcb..24cfcc9 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendALSTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendALSTest.java
@@ -19,9 +19,7 @@
package org.apache.nemo.compiler.frontend.beam;
import org.apache.nemo.client.JobLauncher;
-import org.apache.nemo.common.dag.DAG;
import org.apache.nemo.common.ir.IRDAG;
-import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.compiler.CompilerTestUtil;
import org.junit.Test;
@@ -44,7 +42,7 @@ public final class BeamFrontendALSTest {
assertEquals(producedDAG.getTopologicalSort(), producedDAG.getTopologicalSort());
assertEquals(44, producedDAG.getVertices().size());
-// producedDAG.getTopologicalSort().forEach(v -> System.out.println(v.getId()));
+ // producedDAG.getTopologicalSort().forEach(v -> System.out.println(v.getId()));
final IRVertex vertexX = producedDAG.getTopologicalSort().get(5);
assertEquals(1, producedDAG.getIncomingEdgesOf(vertexX).size());
assertEquals(1, producedDAG.getIncomingEdgesOf(vertexX.getId()).size());
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendMLRTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendMLRTest.java
index 231d9ea..d250485 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendMLRTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/BeamFrontendMLRTest.java
@@ -19,9 +19,7 @@
package org.apache.nemo.compiler.frontend.beam;
import org.apache.nemo.client.JobLauncher;
-import org.apache.nemo.common.dag.DAG;
import org.apache.nemo.common.ir.IRDAG;
-import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.compiler.CompilerTestUtil;
import org.junit.Test;
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java
index 702ca9d..c04875d 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java
@@ -21,7 +21,6 @@ package org.apache.nemo.compiler.frontend.beam.transform;
import org.apache.beam.sdk.transforms.Materialization;
import org.apache.beam.sdk.transforms.Materializations;
import org.apache.beam.sdk.transforms.ViewFn;
-import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -32,7 +31,7 @@ import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Test;
-import java.util.*;
+import java.util.Arrays;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java
index fa1169c..f343f4d 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java
@@ -43,9 +43,7 @@ import org.junit.Test;
import java.util.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
public final class DoFnTransformTest {
@@ -342,6 +340,7 @@ public final class DoFnTransformTest {
/**
* Identitiy do fn.
+ *
* @param <T> type
*/
private static class IdentityDoFn<T> extends DoFn<T, T> {
@@ -354,6 +353,7 @@ public final class DoFnTransformTest {
/**
* Side input do fn.
+ *
* @param <T> type
*/
private static class SimpleSideInputDoFn<T> extends DoFn<T, String> {
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransformTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransformTest.java
index f0749c0..4e51525 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransformTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransformTest.java
@@ -38,9 +38,7 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-import static org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing.EARLY;
-import static org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing.LATE;
-import static org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing.ON_TIME;
+import static org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing.*;
import static org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode.ACCUMULATING_FIRED_PANES;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
@@ -175,7 +173,7 @@ public final class GroupByKeyAndWindowDoFnTransformTest {
doFnTransform.onWatermark(watermark2);
assertEquals(0, oc.outputs.size()); // do not emit anything
- assertEquals(1, oc.watermarks.size());
+ assertEquals(1, oc.watermarks.size());
// check output watermark
assertEquals(1400,
@@ -284,7 +282,7 @@ public final class GroupByKeyAndWindowDoFnTransformTest {
outputTag,
WindowingStrategy.of(window).withTrigger(trigger)
.withMode(ACCUMULATING_FIRED_PANES)
- .withAllowedLateness(lateness),
+ .withAllowedLateness(lateness),
PipelineOptionsFactory.as(NemoPipelineOptions.class),
SystemReduceFn.buffering(NULL_INPUT_CODER),
DisplayData.none());
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/TestOutputCollector.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/TestOutputCollector.java
index 9abff71..50f64df 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/TestOutputCollector.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/TestOutputCollector.java
@@ -28,6 +28,7 @@ import java.util.List;
/**
* Test output collector that collects data and watermarks.
+ *
* @param <T>
*/
final class TestOutputCollector<T> implements OutputCollector<WindowedValue<T>> {
@@ -43,8 +44,8 @@ final class TestOutputCollector<T> implements OutputCollector<WindowedValue<T>>
@Override
public void emit(WindowedValue<T> output) {
- outputs.add(output);
- }
+ outputs.add(output);
+ }
@Override
public void emitWatermark(Watermark watermark) {
@@ -64,6 +65,6 @@ final class TestOutputCollector<T> implements OutputCollector<WindowedValue<T>>
}
public List<Tuple<String, WindowedValue<T>>> getTaggedOutputs() {
- return taggedOutputs;
- }
+ return taggedOutputs;
+ }
}
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeCoderPassTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeCoderPassTest.java
index 88343af..8626222 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeCoderPassTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultEdgeCoderPassTest.java
@@ -21,12 +21,10 @@ package org.apache.nemo.compiler.optimizer.pass.compiletime.annotating;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.coder.DecoderFactory;
import org.apache.nemo.common.coder.EncoderFactory;
-import org.apache.nemo.common.dag.DAG;
import org.apache.nemo.common.ir.IRDAG;
import org.apache.nemo.common.ir.edge.IREdge;
import org.apache.nemo.common.ir.edge.executionproperty.DecoderProperty;
import org.apache.nemo.common.ir.edge.executionproperty.EncoderProperty;
-import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.compiler.CompilerTestUtil;
import org.junit.Before;
import org.junit.Test;
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPassTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPassTest.java
index b92df25..ae2d2e6 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPassTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultParallelismPassTest.java
@@ -56,7 +56,7 @@ public class DefaultParallelismPassTest {
final IRDAG processedDAG = new DefaultParallelismPass().apply(compiledDAG);
processedDAG.getTopologicalSort().forEach(irVertex ->
- assertEquals(1, irVertex.getPropertyValue(ParallelismProperty.class).get().longValue()));
+ assertEquals(1, irVertex.getPropertyValue(ParallelismProperty.class).get().longValue()));
}
@Test
@@ -65,8 +65,8 @@ public class DefaultParallelismPassTest {
final IRDAG processedDAG = new DefaultParallelismPass(desiredSourceParallelism, 2).apply(compiledDAG);
processedDAG.getTopologicalSort().stream()
- .filter(irVertex -> irVertex instanceof SourceVertex)
- .forEach(irVertex -> assertEquals(desiredSourceParallelism,
- irVertex.getPropertyValue(ParallelismProperty.class).get().longValue()));
+ .filter(irVertex -> irVertex instanceof SourceVertex)
+ .forEach(irVertex -> assertEquals(desiredSourceParallelism,
+ irVertex.getPropertyValue(ParallelismProperty.class).get().longValue()));
}
}
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPassTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPassTest.java
index e5b4b10..cd28467 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPassTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/optimizer/pass/compiletime/annotating/DefaultScheduleGroupPassTest.java
@@ -29,8 +29,6 @@ import org.apache.nemo.common.ir.edge.executionproperty.DataFlowProperty;
import org.apache.nemo.common.ir.vertex.IRVertex;
import org.apache.nemo.common.ir.vertex.OperatorVertex;
import org.apache.nemo.common.ir.vertex.executionproperty.ScheduleGroupProperty;
-import org.apache.nemo.compiler.CompilerTestUtil;
-import org.apache.nemo.compiler.optimizer.policy.TestPolicy;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
@@ -39,9 +37,7 @@ import org.powermock.modules.junit4.PowerMockRunner;
import java.util.*;
import static org.apache.nemo.common.test.EmptyComponents.EMPTY_TRANSFORM;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
/**
* Test {@link DefaultScheduleGroupPass}.
@@ -58,17 +54,17 @@ public final class DefaultScheduleGroupPassTest {
/**
* Return a DAG that has a branch.
* {@literal
- * /-- v3 --- v4
+ * /-- v3 --- v4
* v0 --- v1 --- v2 --/
* }
*
* @param communicationPattern {@link CommunicationPatternProperty.Value} for the edges
- * @param dataFlowModel {@link DataFlowProperty.Value} for the edges
+ * @param dataFlowModel {@link DataFlowProperty.Value} for the edges
* @return a {@link Pair} of {@link DAG} and {@link List} of {@link IRVertex}
*/
private static Pair<IRDAG, List<IRVertex>> generateBranchDAG(
- final CommunicationPatternProperty.Value communicationPattern,
- final DataFlowProperty.Value dataFlowModel) {
+ final CommunicationPatternProperty.Value communicationPattern,
+ final DataFlowProperty.Value dataFlowModel) {
final DAGBuilder<IRVertex, IREdge> dagBuilder = new DAGBuilder<>();
final IRVertex v0 = new OperatorVertex(EMPTY_TRANSFORM);
@@ -102,12 +98,12 @@ public final class DefaultScheduleGroupPassTest {
* }
*
* @param communicationPattern {@link CommunicationPatternProperty.Value} for the edges
- * @param dataFlowModel {@link DataFlowProperty.Value} for the edges
+ * @param dataFlowModel {@link DataFlowProperty.Value} for the edges
* @return a {@link Pair} of {@link DAG} and {@link List} of {@link IRVertex}
*/
private static Pair<IRDAG, List<IRVertex>> generateJoinDAG(
- final CommunicationPatternProperty.Value communicationPattern,
- final DataFlowProperty.Value dataFlowModel) {
+ final CommunicationPatternProperty.Value communicationPattern,
+ final DataFlowProperty.Value dataFlowModel) {
final DAGBuilder<IRVertex, IREdge> dagBuilder = new DAGBuilder<>();
final IRVertex v0 = new OperatorVertex(EMPTY_TRANSFORM);
@@ -136,8 +132,9 @@ public final class DefaultScheduleGroupPassTest {
/**
* Asserts that the {@link ScheduleGroupProperty} is equal to {@code expected}.
+ *
* @param expected the expected property value
- * @param vertex the vertex to test
+ * @param vertex the vertex to test
*/
private static void assertScheduleGroup(final int expected, final IRVertex vertex) {
assertEquals(expected, getScheduleGroup(vertex));
@@ -149,11 +146,12 @@ public final class DefaultScheduleGroupPassTest {
*/
private static int getScheduleGroup(final IRVertex vertex) {
return vertex.getPropertyValue(ScheduleGroupProperty.class)
- .orElseThrow(() -> new RuntimeException(String.format("ScheduleGroup not set for %s", vertex.getId())));
+ .orElseThrow(() -> new RuntimeException(String.format("ScheduleGroup not set for %s", vertex.getId())));
}
/**
* Ensures that all vertices in {@code vertices} have different {@link ScheduleGroupProperty} value.
+ *
* @param vertices vertices to test
*/
private static void assertDifferentScheduleGroup(final Collection<IRVertex> vertices) {
@@ -173,7 +171,7 @@ public final class DefaultScheduleGroupPassTest {
public void testBranch() {
final DefaultScheduleGroupPass pass = new DefaultScheduleGroupPass();
final Pair<IRDAG, List<IRVertex>> dag
- = generateBranchDAG(CommunicationPatternProperty.Value.OneToOne, DataFlowProperty.Value.Push);
+ = generateBranchDAG(CommunicationPatternProperty.Value.OneToOne, DataFlowProperty.Value.Push);
pass.apply(dag.left());
dag.right().forEach(v -> assertScheduleGroup(0, v));
}
@@ -186,7 +184,7 @@ public final class DefaultScheduleGroupPassTest {
public void testBranchWhenMultipleInEdgeNotAllowed() {
final DefaultScheduleGroupPass pass = new DefaultScheduleGroupPass(false, false, false);
final Pair<IRDAG, List<IRVertex>> dag
- = generateBranchDAG(CommunicationPatternProperty.Value.OneToOne, DataFlowProperty.Value.Pull);
+ = generateBranchDAG(CommunicationPatternProperty.Value.OneToOne, DataFlowProperty.Value.Pull);
pass.apply(dag.left());
dag.right().subList(0, 4).forEach(v -> assertScheduleGroup(0, v));
dag.right().subList(4, 5).forEach(v -> assertScheduleGroup(1, v));
@@ -200,7 +198,7 @@ public final class DefaultScheduleGroupPassTest {
public void testBranchWithPush() {
final DefaultScheduleGroupPass pass = new DefaultScheduleGroupPass(false, false, false);
final Pair<IRDAG, List<IRVertex>> dag
- = generateBranchDAG(CommunicationPatternProperty.Value.Shuffle, DataFlowProperty.Value.Push);
+ = generateBranchDAG(CommunicationPatternProperty.Value.Shuffle, DataFlowProperty.Value.Push);
pass.apply(dag.left());
dag.right().forEach(v -> assertScheduleGroup(0, v));
}
@@ -213,7 +211,7 @@ public final class DefaultScheduleGroupPassTest {
public void testBranchWithBroadcast() {
final DefaultScheduleGroupPass pass = new DefaultScheduleGroupPass(false, true, true);
... 14180 lines suppressed ...