You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by ja...@apache.org on 2018/08/08 02:37:39 UTC
[incubator-nemo] 02/02: Merge branch 'master' into nemo-175
This is an automated email from the ASF dual-hosted git repository.
jangho pushed a commit to branch nemo-175
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit eb01821753a5cbe9aca37a9243393ff9b602455d
Merge: 1a43e79 a749280
Author: Jangho Seo <ja...@jangho.io>
AuthorDate: Wed Aug 8 11:37:28 2018 +0900
Merge branch 'master' into nemo-175
.../ResourceLocalityProperty.java | 3 +-
.../main/java/edu/snu/nemo/common/pass/Pass.java | 64 ++++++++++
.../compiler/backend/nemo/NemoBackendTest.java | 3 +-
.../MapReduceDisaggregationOptimization.java | 3 +-
.../pass/compiletime/CompileTimePass.java | 8 +-
.../compiletime/annotating/AnnotatingPass.java | 2 +-
.../pass/compiletime/composite/CompositePass.java | 3 +-
.../pass/compiletime/reshaping/ReshapingPass.java | 5 +-
.../compiler/optimizer/policy/BasicPullPolicy.java | 33 +++--
.../compiler/optimizer/policy/BasicPushPolicy.java | 35 ++++--
.../policy/ConditionalLargeShufflePolicy.java | 68 ++++++++++
.../compiler/optimizer/policy/DataSkewPolicy.java | 30 +++--
.../compiler/optimizer/policy/DefaultPolicy.java | 25 ++--
.../policy/DefaultPolicyWithSeparatePass.java | 28 +++--
.../optimizer/policy/DisaggregationPolicy.java | 29 +++--
.../optimizer/policy/LargeShufflePolicy.java | 29 +++--
.../snu/nemo/compiler/optimizer/policy/Policy.java | 22 ++--
.../compiler/optimizer/policy/PolicyBuilder.java | 78 ++++++++----
.../PolicyImpl.java} | 95 +++++++++-----
.../optimizer/policy/TransientResourcePolicy.java | 29 +++--
.../optimizer/policy/PolicyBuilderTest.java | 15 +--
.../nemo/compiler/optimizer/policy/TestPolicy.java | 32 +++--
.../compiler/backend/nemo/DAGConverterTest.java | 9 +-
.../annotating/DefaultScheduleGroupPassTest.java | 5 +-
.../nemo/examples/beam/PerPercentileAverage.java | 138 +++++++++++++++++++++
...ITCase.java => PerPercentileAverageITCase.java} | 58 +++------
.../snu/nemo/examples/beam/WordCountITCase.java | 12 +-
.../beam/policy/DataSkewPolicyParallelismFive.java | 23 ++--
.../beam/policy/DefaultPolicyParallelismFive.java | 23 ++--
.../DisaggregationPolicyParallelismFive.java | 25 ++--
.../policy/LargeShufflePolicyParallelismFive.java | 26 ++--
.../nemo/examples/beam/policy/PolicyTestUtil.java | 34 ++---
.../TransientResourcePolicyParallelismFive.java | 25 ++--
.../TransientResourcePolicyParallelismTen.java | 25 ++--
examples/resources/expected_output_partition_0 | 1 +
examples/resources/expected_output_partition_1 | 1 +
examples/resources/expected_output_partition_2 | 1 +
examples/resources/expected_output_partition_3 | 1 +
examples/resources/expected_output_partition_4 | 1 +
examples/resources/expected_output_partition_5 | 1 +
examples/resources/expected_output_partition_6 | 1 +
examples/resources/expected_output_partition_7 | 0
examples/resources/expected_output_partition_8 | 1 +
examples/resources/expected_output_partition_9 | 1 +
examples/resources/test_input_partition | 28 +++++
.../DynamicOptimizationEventHandler.java | 4 +-
...RuntimeOptimizer.java => RunTimeOptimizer.java} | 4 +-
.../pass/runtime/DataSkewRuntimePass.java | 4 +-
.../common/optimizer/pass/runtime/RuntimePass.java | 8 +-
.../edu/snu/nemo/driver/UserApplicationRunner.java | 19 +--
.../executor/task/ParentTaskDataFetcher.java | 17 ++-
.../executor/task/ParentTaskDataFetcherTest.java | 48 ++++++-
.../SourceLocationAwareSchedulingConstraint.java | 82 +++++++++---
.../SchedulingConstraintnRegistryTest.java | 14 ++-
...ourceLocationAwareSchedulingConstraintTest.java | 16 ++-
.../runtime/common/plan/TestPlanGenerator.java | 3 +-
56 files changed, 912 insertions(+), 386 deletions(-)
diff --cc runtime/common/src/main/java/edu/snu/nemo/runtime/common/eventhandler/DynamicOptimizationEventHandler.java
index 190e9ee,f4f7d22..56ce7c1
--- a/runtime/common/src/main/java/edu/snu/nemo/runtime/common/eventhandler/DynamicOptimizationEventHandler.java
+++ b/runtime/common/src/main/java/edu/snu/nemo/runtime/common/eventhandler/DynamicOptimizationEventHandler.java
@@@ -53,10 -54,11 +53,10 @@@ public final class DynamicOptimizationE
final MetricCollectionBarrierVertex metricCollectionBarrierVertex =
dynamicOptimizationEvent.getMetricCollectionBarrierVertex();
- final PhysicalPlan newPlan = RuntimeOptimizer.dynamicOptimization(physicalPlan,
- final Pair<String, String> taskInfo = dynamicOptimizationEvent.getTaskInfo();
-
+ final PhysicalPlan newPlan = RunTimeOptimizer.dynamicOptimization(physicalPlan,
metricCollectionBarrierVertex);
- pubSubEventHandler.onNext(new UpdatePhysicalPlanEvent(newPlan, taskInfo));
+ pubSubEventHandler.onNext(new UpdatePhysicalPlanEvent(newPlan, dynamicOptimizationEvent.getTaskId(),
+ dynamicOptimizationEvent.getExecutorId()));
}
}