You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by jo...@apache.org on 2018/11/01 01:54:06 UTC
[incubator-nemo] 02/02: Merge branch 'master' into maven-artifact
This is an automated email from the ASF dual-hosted git repository.
jooykim pushed a commit to branch maven-artifact
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit 49266b5b70aa7bc317807d20e2dd7a1cfb170ffd
Merge: b2787ca eea3f57
Author: Joo Yeon Kim <jo...@apache.org>
AuthorDate: Wed Oct 31 11:05:38 2018 +0900
Merge branch 'master' into maven-artifact
.gitignore | 1 +
.travis.yml | 4 +-
checkstyle.xml | 2 +-
.../java/org/apache/nemo/client/JobLauncher.java | 15 +-
...Collector.java => BoundedIteratorReadable.java} | 43 ++--
.../org/apache/nemo/common/ir/OutputCollector.java | 7 +
.../java/org/apache/nemo/common/ir/Readable.java | 38 +++-
.../edge/executionproperty/DataStoreProperty.java | 1 +
.../nemo/common/ir/vertex/CachedSourceVertex.java | 40 +++-
.../common/ir/vertex/InMemorySourceVertex.java | 25 ++-
.../apache/nemo/common/ir/vertex/SourceVertex.java | 2 +
.../vertex/transform/AggregateMetricTransform.java | 2 +-
.../vertex/transform/MetricCollectTransform.java | 2 +-
.../vertex/transform/NoWatermarkEmitTransform.java | 24 +--
.../common/ir/vertex/transform/RelayTransform.java | 6 +
.../nemo/common/ir/vertex/transform/Transform.java | 10 +
.../apache/nemo/common/punctuation/Finishmark.java | 23 +--
.../Watermark.java} | 48 +++--
.../org/apache/nemo/common/test/ArgBuilder.java | 9 +
.../apache/nemo/common/test/EmptyComponents.java | 37 +++-
.../compiler/frontend/beam/PipelineTranslator.java | 21 +-
.../beam/source/BeamBoundedSourceVertex.java | 73 +++++--
.../beam/source/BeamUnboundedSourceVertex.java | 128 ++++++------
.../beam/transform/AbstractDoFnTransform.java | 6 +-
.../beam/transform/CreateViewTransform.java | 4 +-
.../frontend/beam/transform/DoFnTransform.java | 8 +
.../frontend/beam/transform/FlattenTransform.java | 6 +
.../GroupByKeyAndWindowDoFnTransform.java | 10 +-
.../beam/transform/GroupByKeyTransform.java | 4 +-
.../frontend/beam/transform/WindowFnTransform.java | 6 +
compiler/frontend/spark/pom.xml | 19 ++
.../source/SparkDatasetBoundedSourceVertex.java | 29 ++-
.../source/SparkTextFileBoundedSourceVertex.java | 34 +++-
.../frontend/spark/transform/CollectTransform.java | 5 +-
.../frontend/spark/transform/FlatMapTransform.java | 6 +
.../spark/transform/GroupByKeyTransform.java | 3 +-
.../spark/transform/HDFSTextFileTransform.java | 3 +-
.../spark/transform/LocalTextFileTransform.java | 4 +-
.../spark/transform/MapToPairTransform.java | 6 +
.../frontend/spark/transform/MapTransform.java | 6 +
.../spark/transform/ReduceByKeyTransform.java | 4 +-
.../frontend/spark/transform/ReduceTransform.java | 6 +
.../annotating/PipeTransferForAllEdgesPass.java | 41 ++--
.../frontend/beam/transform/DoFnTransformTest.java | 6 +
.../main/java/org/apache/nemo/conf/JobConf.java | 8 +
.../nemo/examples/beam/WriteOneFilePerWindow.java | 2 +-
.../examples/beam/WindowedWordCountITCase.java | 48 ++++-
.../policy/StreamingPolicyParallelismFive.java | 57 ++++++
pom.xml | 9 +
.../runtime/common/message/MessageEnvironment.java | 1 +
.../common/message/ncs/NcsMessageEnvironment.java | 7 +
.../nemo/runtime/common/plan/RuntimeEdge.java | 15 ++
runtime/common/src/main/proto/ControlMessage.proto | 33 +++-
.../org/apache/nemo/runtime/executor/Executor.java | 10 +-
.../executor/bytetransfer/ByteTransfer.java | 12 +-
.../executor/bytetransfer/ByteTransferContext.java | 10 +-
.../ByteTransportChannelInitializer.java | 11 +-
.../bytetransfer/ClosableBlockingQueue.java | 1 +
.../executor/bytetransfer/ContextManager.java | 40 +++-
.../executor/bytetransfer/ControlFrameEncoder.java | 1 +
.../runtime/executor/data/BlockManagerWorker.java | 10 +-
.../executor/data/BroadcastManagerWorker.java | 3 +-
.../nemo/runtime/executor/data/PipeContainer.java | 150 ++++++++++++++
.../runtime/executor/data/PipeManagerWorker.java | 199 +++++++++++++++++++
.../runtime/executor/data/SerializerManager.java | 2 +-
.../data/partitioner/DataSkewHashPartitioner.java | 3 -
.../executor/data/stores/GlusterFileStore.java | 3 -
.../executor/data/stores/LocalFileStore.java | 3 -
.../{InputReader.java => BlockInputReader.java} | 125 ++++--------
.../{OutputWriter.java => BlockOutputWriter.java} | 90 +++------
.../datatransfer/DataFetcherOutputCollector.java | 6 +
.../datatransfer/DynOptDataOutputCollector.java | 6 +
.../runtime/executor/datatransfer/InputReader.java | 155 +--------------
...Factory.java => IntermediateDataIOFactory.java} | 35 +++-
.../OperatorVertexOutputCollector.java | 20 ++
.../executor/datatransfer/OutputWriter.java | 158 ++++-----------
.../executor/datatransfer/PipeInputReader.java | 82 ++++++++
.../executor/datatransfer/PipeOutputWriter.java | 136 +++++++++++++
.../nemo/runtime/executor/task/DataFetcher.java | 5 +-
.../executor/task/ParentTaskDataFetcher.java | 12 +-
.../executor/task/SourceVertexDataFetcher.java | 81 ++++++--
.../nemo/runtime/executor/task/TaskExecutor.java | 203 ++++++++++++++-----
.../executor/datatransfer/DataTransferTest.java | 77 +++++---
.../executor/task/ParentTaskDataFetcherTest.java | 13 +-
.../runtime/executor/task/TaskExecutorTest.java | 219 +++++++++++++++++++--
.../apache/nemo/runtime/master/MetricStore.java | 2 +
.../nemo/runtime/master/PipeManagerMaster.java | 157 +++++++++++++++
.../apache/nemo/runtime/master/RuntimeMaster.java | 1 +
.../master/resource/ExecutorRepresenter.java | 6 +-
.../master/resource/ResourceSpecification.java | 2 +-
.../nemo/runtime/master/scheduler/Scheduler.java | 2 -
.../master/scheduler/StreamingScheduler.java | 29 ++-
.../runtime/master/scheduler/TaskDispatcher.java | 10 +-
.../master/scheduler/StreamingSchedulerTest.java | 8 +-
.../runtime/master/scheduler/TaskRetryTest.java | 2 +-
95 files changed, 2214 insertions(+), 843 deletions(-)