You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by se...@apache.org on 2015/04/18 00:37:58 UTC
incubator-reef git commit: [REEF-268] Simplify Configuration
passthrough from Client to Driver to Evaluator
Repository: incubator-reef
Updated Branches:
refs/heads/master c330dcff5 -> 774a5d75e
[REEF-268] Simplify Configuration passthrough from Client to Driver to Evaluator
This simplifies the passthrough of Configuration from the Client to the
Driver to the Evaluators. To do so, it adds the new interface
`ConfigurationProvider` for classes that provide Configurations. This
also adds two new named sets:
* `DriverConfigurationProviders` for `ConfigurationProvider`s to be
considered when producing the Driver Configuration in the Client.
* `EvaluatorConfigurationProviders` for `ConfigurationProvider`s to be
considered when producing the Evaluator Configuration in the Driver.
This change also adds a new common configuration module to be merged
into all runtime client configuration modules,
`CommonRuntimeConfiguration`. For now, this only binds the basic REEF
implementations shared by all runtimes. It is included in this change
because it provides a good handle for us to add `ConfigurationProvider`s
to `DriverConfigurationProviders` in the future, e.g. in [REEF-246] and
[REEF-247].
JIRA:
[REEF-268](https://issues.apache.org/jira/browse/REEF-268)
Pull Request:
This closes #150
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/774a5d75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/774a5d75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/774a5d75
Branch: refs/heads/master
Commit: 774a5d75e60d091c28014f497a9b9c1789fa8c10
Parents: c330dcf
Author: Markus Weimer <we...@apache.org>
Authored: Fri Apr 17 14:19:27 2015 -0700
Committer: Beysim Sezgin <be...@microsoft.com>
Committed: Fri Apr 17 15:35:36 2015 -0700
----------------------------------------------------------------------
.../DriverConfigurationProviders.java | 32 ++++++++++++++++
.../reef/common/ConfigurationProvider.java | 32 ++++++++++++++++
.../EvaluatorConfigurationProviders.java | 32 ++++++++++++++++
.../client/CommonRuntimeConfiguration.java | 39 ++++++++++++++++++++
.../common/client/REEFImplementation.java | 35 ++++++++++++++++--
.../evaluator/AllocatedEvaluatorImpl.java | 30 +++++++++++++--
.../driver/evaluator/EvaluatorManager.java | 10 +++--
.../HDInsightRuntimeConfigurationStatic.java | 13 ++-----
.../local/client/LocalRuntimeConfiguration.java | 23 ++++++------
.../mesos/client/MesosClientConfiguration.java | 13 ++-----
.../yarn/client/YarnClientConfiguration.java | 24 ++++++------
11 files changed, 230 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/client/parameters/DriverConfigurationProviders.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/client/parameters/DriverConfigurationProviders.java b/lang/java/reef-common/src/main/java/org/apache/reef/client/parameters/DriverConfigurationProviders.java
new file mode 100644
index 0000000..85cee64
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/client/parameters/DriverConfigurationProviders.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.client.parameters;
+
+import org.apache.reef.common.ConfigurationProvider;
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+import java.util.Set;
+
+/**
+ * Configuration provides whose Configurations will be mixed into the Driver Configuration.
+ */
+@NamedParameter(doc = "Configuration provides whose Configurations will be mixed into the Driver Configuration.")
+public final class DriverConfigurationProviders implements Name<Set<ConfigurationProvider>> {
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/common/ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/common/ConfigurationProvider.java b/lang/java/reef-common/src/main/java/org/apache/reef/common/ConfigurationProvider.java
new file mode 100644
index 0000000..7eeb42d
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/common/ConfigurationProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.common;
+
+import org.apache.reef.tang.Configuration;
+
+/**
+ * Objects that can provide a Configuration implement this interface.
+ */
+public interface ConfigurationProvider {
+
+ /**
+ * @return a Configuration provided by this instance.
+ */
+ Configuration getConfiguration();
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/driver/parameters/EvaluatorConfigurationProviders.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/driver/parameters/EvaluatorConfigurationProviders.java b/lang/java/reef-common/src/main/java/org/apache/reef/driver/parameters/EvaluatorConfigurationProviders.java
new file mode 100644
index 0000000..a1dbff8
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/driver/parameters/EvaluatorConfigurationProviders.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.driver.parameters;
+
+import org.apache.reef.common.ConfigurationProvider;
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+import java.util.Set;
+
+/**
+ * Configuration provides whose Configurations will be mixed into the Evaluator Configuration.
+ */
+@NamedParameter(doc = "Configuration provides whose Configurations will be mixed into the Evaluator Configuration.")
+public final class EvaluatorConfigurationProviders implements Name<Set<ConfigurationProvider>> {
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/CommonRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/CommonRuntimeConfiguration.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/CommonRuntimeConfiguration.java
new file mode 100644
index 0000000..5440429
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/CommonRuntimeConfiguration.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.common.client;
+
+import org.apache.reef.annotations.audience.RuntimeAuthor;
+import org.apache.reef.client.REEF;
+import org.apache.reef.client.RunningJob;
+import org.apache.reef.runtime.common.launch.REEFMessageCodec;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.wake.remote.RemoteConfiguration;
+
+/**
+ * A ConfigurationModule to be merged in by all runtimes into their client configuration.
+ */
+@RuntimeAuthor
+public final class CommonRuntimeConfiguration extends ConfigurationModuleBuilder {
+ public static final ConfigurationModule CONF = new CommonRuntimeConfiguration()
+ .bindImplementation(REEF.class, REEFImplementation.class)
+ .bindImplementation(RunningJob.class, RunningJobImpl.class)
+ .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .build();
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/REEFImplementation.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/REEFImplementation.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/REEFImplementation.java
index f0074f9..cb9f982 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/REEFImplementation.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/client/REEFImplementation.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,17 +22,24 @@ import org.apache.reef.annotations.Provided;
import org.apache.reef.annotations.audience.ClientSide;
import org.apache.reef.annotations.audience.Private;
import org.apache.reef.client.REEF;
+import org.apache.reef.client.parameters.DriverConfigurationProviders;
+import org.apache.reef.common.ConfigurationProvider;
import org.apache.reef.proto.ClientRuntimeProtocol.JobSubmissionProto;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
import org.apache.reef.runtime.common.launch.parameters.ErrorHandlerRID;
import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.ConfigurationBuilder;
+import org.apache.reef.tang.Tang;
import org.apache.reef.tang.annotations.Name;
import org.apache.reef.tang.annotations.NamedParameter;
+import org.apache.reef.tang.annotations.Parameter;
import org.apache.reef.util.REEFVersion;
+import org.apache.reef.util.logging.Config;
import org.apache.reef.util.logging.LoggingScope;
import org.apache.reef.util.logging.LoggingScopeFactory;
import javax.inject.Inject;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -48,6 +55,7 @@ public final class REEFImplementation implements REEF {
private final JobSubmissionHelper jobSubmissionHelper;
private final ClientWireUp clientWireUp;
private final LoggingScopeFactory loggingScopeFactory;
+ private final Set<ConfigurationProvider> configurationProviders;
/**
* @param jobSubmissionHandler
@@ -56,6 +64,7 @@ public final class REEFImplementation implements REEF {
* @param jobStatusMessageHandler is passed only to make sure it is instantiated
* @param clientWireUp
* @param reefVersion provides the current version of REEF.
+ * @param configurationProviders
*/
@Inject
REEFImplementation(final JobSubmissionHandler jobSubmissionHandler,
@@ -64,11 +73,13 @@ public final class REEFImplementation implements REEF {
final JobStatusMessageHandler jobStatusMessageHandler,
final ClientWireUp clientWireUp,
final LoggingScopeFactory loggingScopeFactory,
- final REEFVersion reefVersion) {
+ final REEFVersion reefVersion,
+ final @Parameter(DriverConfigurationProviders.class) Set<ConfigurationProvider> configurationProviders) {
this.jobSubmissionHandler = jobSubmissionHandler;
this.runningJobs = runningJobs;
this.jobSubmissionHelper = jobSubmissionHelper;
this.clientWireUp = clientWireUp;
+ this.configurationProviders = configurationProviders;
clientWireUp.performWireUp();
this.loggingScopeFactory = loggingScopeFactory;
reefVersion.logVersion();
@@ -83,14 +94,15 @@ public final class REEFImplementation implements REEF {
@Override
public void submit(final Configuration driverConf) {
try (LoggingScope ls = this.loggingScopeFactory.reefSubmit()) {
+ final Configuration driverConfiguration = createDriverConfiguration(driverConf);
final JobSubmissionProto submissionMessage;
try {
if (this.clientWireUp.isClientPresent()) {
- submissionMessage = this.jobSubmissionHelper.getJobsubmissionProto(driverConf)
+ submissionMessage = this.jobSubmissionHelper.getJobsubmissionProto(driverConfiguration)
.setRemoteId(this.clientWireUp.getRemoteManagerIdentifier())
.build();
} else {
- submissionMessage = this.jobSubmissionHelper.getJobsubmissionProto(driverConf)
+ submissionMessage = this.jobSubmissionHelper.getJobsubmissionProto(driverConfiguration)
.setRemoteId(ErrorHandlerRID.NONE)
.build();
}
@@ -102,6 +114,21 @@ public final class REEFImplementation implements REEF {
}
}
+ /**
+ * Assembles the final Driver Configuration by merging in all the Configurations provided by ConfigurationProviders.
+ *
+ * @param driverConfiguration
+ * @return
+ */
+ private Configuration createDriverConfiguration(final Configuration driverConfiguration) {
+ final ConfigurationBuilder configurationBuilder = Tang.Factory.getTang()
+ .newConfigurationBuilder(driverConfiguration);
+ for (final ConfigurationProvider configurationProvider : this.configurationProviders) {
+ configurationBuilder.addConfiguration(configurationProvider.getConfiguration());
+ }
+ return configurationBuilder.build();
+ }
+
@NamedParameter(doc = "The driver remote identifier.")
public final static class DriverRemoteIdentifier implements Name<String> {
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/AllocatedEvaluatorImpl.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/AllocatedEvaluatorImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/AllocatedEvaluatorImpl.java
index 462d6c9..754cd3a 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/AllocatedEvaluatorImpl.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/AllocatedEvaluatorImpl.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,6 +20,7 @@ package org.apache.reef.runtime.common.driver.evaluator;
import org.apache.reef.annotations.audience.DriverSide;
import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.common.ConfigurationProvider;
import org.apache.reef.driver.context.ContextConfiguration;
import org.apache.reef.driver.evaluator.AllocatedEvaluator;
import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
@@ -28,6 +29,8 @@ import org.apache.reef.proto.DriverRuntimeProtocol;
import org.apache.reef.proto.ReefServiceProtos;
import org.apache.reef.runtime.common.evaluator.EvaluatorConfiguration;
import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.ConfigurationBuilder;
+import org.apache.reef.tang.Tang;
import org.apache.reef.tang.exceptions.BindException;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationSerializer;
@@ -38,6 +41,7 @@ import org.apache.reef.util.logging.LoggingScopeFactory;
import java.io.File;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -55,6 +59,7 @@ final class AllocatedEvaluatorImpl implements AllocatedEvaluator {
private final ConfigurationSerializer configurationSerializer;
private final String jobIdentifier;
private final LoggingScopeFactory loggingScopeFactory;
+ private final Set<ConfigurationProvider> evaluatorConfigurationProviders;
/**
* The set of files to be places on the Evaluator.
@@ -69,12 +74,14 @@ final class AllocatedEvaluatorImpl implements AllocatedEvaluator {
final String remoteID,
final ConfigurationSerializer configurationSerializer,
final String jobIdentifier,
- final LoggingScopeFactory loggingScopeFactory) {
+ final LoggingScopeFactory loggingScopeFactory,
+ final Set<ConfigurationProvider> evaluatorConfigurationProviders) {
this.evaluatorManager = evaluatorManager;
this.remoteID = remoteID;
this.configurationSerializer = configurationSerializer;
this.jobIdentifier = jobIdentifier;
this.loggingScopeFactory = loggingScopeFactory;
+ this.evaluatorConfigurationProviders = evaluatorConfigurationProviders;
}
@Override
@@ -146,12 +153,13 @@ final class AllocatedEvaluatorImpl implements AllocatedEvaluator {
final Optional<Configuration> taskConfiguration) {
try (final LoggingScope lb = loggingScopeFactory.evaluatorLaunch(this.getId())) {
try {
+ final Configuration rootContextConfiguration = makeRootContextConfiguration(contextConfiguration);
final ConfigurationModule evaluatorConfigurationModule = EvaluatorConfiguration.CONF
.set(EvaluatorConfiguration.APPLICATION_IDENTIFIER, this.jobIdentifier)
.set(EvaluatorConfiguration.DRIVER_REMOTE_IDENTIFIER, this.remoteID)
.set(EvaluatorConfiguration.EVALUATOR_IDENTIFIER, this.getId());
- final String encodedContextConfigurationString = this.configurationSerializer.toString(contextConfiguration);
+ final String encodedContextConfigurationString = this.configurationSerializer.toString(rootContextConfiguration);
// Add the (optional) service configuration
final ConfigurationModule contextConfigurationModule;
if (serviceConfiguration.isPresent()) {
@@ -217,6 +225,22 @@ final class AllocatedEvaluatorImpl implements AllocatedEvaluator {
}
}
+ /**
+ * Merges the Configurations provided by the evaluatorConfigurationProviders into the given
+ * contextConfiguration.
+ *
+ * @param contextConfiguration
+ * @return
+ */
+ private Configuration makeRootContextConfiguration(final Configuration contextConfiguration) {
+ final ConfigurationBuilder configurationBuilder = Tang.Factory.getTang()
+ .newConfigurationBuilder(contextConfiguration);
+ for (final ConfigurationProvider configurationProvider : this.evaluatorConfigurationProviders) {
+ configurationBuilder.addConfiguration(configurationProvider.getConfiguration());
+ }
+ return configurationBuilder.build();
+ }
+
@Override
public String toString() {
return "AllocatedEvaluator{ID='" + getId() + "\'}";
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManager.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManager.java
index a257234..241892a 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManager.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManager.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,11 +20,13 @@ package org.apache.reef.runtime.common.driver.evaluator;
import org.apache.reef.annotations.audience.DriverSide;
import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.common.ConfigurationProvider;
import org.apache.reef.driver.context.ActiveContext;
import org.apache.reef.driver.context.FailedContext;
import org.apache.reef.driver.evaluator.AllocatedEvaluator;
import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
import org.apache.reef.driver.evaluator.EvaluatorType;
+import org.apache.reef.driver.parameters.EvaluatorConfigurationProviders;
import org.apache.reef.driver.task.FailedTask;
import org.apache.reef.exception.EvaluatorException;
import org.apache.reef.exception.EvaluatorKilledByResourceManagerException;
@@ -56,6 +58,7 @@ import org.apache.reef.wake.time.event.Alarm;
import javax.inject.Inject;
import java.io.File;
import java.util.List;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -115,7 +118,8 @@ public final class EvaluatorManager implements Identifiable, AutoCloseable {
final DriverStatusManager driverStatusManager,
final ExceptionCodec exceptionCodec,
final EventHandlerIdlenessSource idlenessSource,
- final LoggingScopeFactory loggingScopeFactory) {
+ final LoggingScopeFactory loggingScopeFactory,
+ final @Parameter(EvaluatorConfigurationProviders.class) Set<ConfigurationProvider> evaluatorConfigurationProviders) {
this.contextRepresenters = contextRepresenters;
this.idlenessSource = idlenessSource;
LOG.log(Level.FINEST, "Instantiating 'EvaluatorManager' for evaluator: {0}", evaluatorId);
@@ -134,7 +138,7 @@ public final class EvaluatorManager implements Identifiable, AutoCloseable {
this.loggingScopeFactory = loggingScopeFactory;
final AllocatedEvaluator allocatedEvaluator =
- new AllocatedEvaluatorImpl(this, remoteManager.getMyIdentifier(), configurationSerializer, getJobIdentifier(), loggingScopeFactory);
+ new AllocatedEvaluatorImpl(this, remoteManager.getMyIdentifier(), configurationSerializer, getJobIdentifier(), loggingScopeFactory, evaluatorConfigurationProviders);
LOG.log(Level.FINEST, "Firing AllocatedEvaluator event for Evaluator with ID [{0}]", evaluatorId);
this.messageDispatcher.onEvaluatorAllocated(allocatedEvaluator);
LOG.log(Level.FINEST, "Instantiated 'EvaluatorManager' for evaluator: [{0}]", this.getId());
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
index e84e77f..babb846 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,19 +19,14 @@
package org.apache.reef.runtime.hdinsight.client;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.reef.client.REEF;
-import org.apache.reef.client.RunningJob;
-import org.apache.reef.runtime.common.client.REEFImplementation;
-import org.apache.reef.runtime.common.client.RunningJobImpl;
+import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
-import org.apache.reef.runtime.common.launch.REEFMessageCodec;
import org.apache.reef.runtime.hdinsight.HDInsightClasspathProvider;
import org.apache.reef.runtime.hdinsight.client.sslhacks.DefaultClientConstructor;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
import org.apache.reef.util.logging.LoggingSetup;
-import org.apache.reef.wake.remote.RemoteConfiguration;
/**
* The static part of the HDInsightRuntimeConfiguration.
@@ -42,9 +37,7 @@ public final class HDInsightRuntimeConfigurationStatic extends ConfigurationModu
}
public static final ConfigurationModule CONF = new HDInsightRuntimeConfigurationStatic()
- .bindImplementation(REEF.class, REEFImplementation.class)
- .bindImplementation(RunningJob.class, RunningJobImpl.class)
- .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .merge(CommonRuntimeConfiguration.CONF)
.bindImplementation(JobSubmissionHandler.class, HDInsightJobSubmissionHandler.class)
.bindConstructor(CloseableHttpClient.class, DefaultClientConstructor.class)
.bindImplementation(RuntimeClasspathProvider.class, HDInsightClasspathProvider.class)
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java
index a5d94dd..55e305b 100644
--- a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java
+++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,21 +18,19 @@
*/
package org.apache.reef.runtime.local.client;
-import org.apache.reef.client.REEF;
-import org.apache.reef.client.RunningJob;
-import org.apache.reef.runtime.common.client.REEFImplementation;
-import org.apache.reef.runtime.common.client.RunningJobImpl;
+import org.apache.reef.client.parameters.DriverConfigurationProviders;
+import org.apache.reef.common.ConfigurationProvider;
+import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
-import org.apache.reef.runtime.common.launch.REEFMessageCodec;
import org.apache.reef.runtime.common.parameters.JVMHeapSlack;
import org.apache.reef.runtime.local.LocalClasspathProvider;
import org.apache.reef.runtime.local.client.parameters.MaxNumberOfEvaluators;
import org.apache.reef.runtime.local.client.parameters.RootFolder;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.tang.formats.OptionalImpl;
import org.apache.reef.tang.formats.OptionalParameter;
-import org.apache.reef.wake.remote.RemoteConfiguration;
import java.util.concurrent.ExecutorService;
@@ -63,18 +61,21 @@ public class LocalRuntimeConfiguration extends ConfigurationModuleBuilder {
public static final OptionalParameter<Double> JVM_HEAP_SLACK = new OptionalParameter<>();
/**
+ * Configuration provides whose Configuration will be merged into all Driver Configuration.
+ */
+ public static final OptionalImpl<ConfigurationProvider> DRIVER_CONFIGURATION_PROVIDERS = new OptionalImpl<>();
+
+ /**
* The ConfigurationModule for the local resourcemanager.
*/
public static final ConfigurationModule CONF = new LocalRuntimeConfiguration()
- .bindImplementation(REEF.class, REEFImplementation.class)
- .bindImplementation(RunningJob.class, RunningJobImpl.class)
+ .merge(CommonRuntimeConfiguration.CONF)
.bindImplementation(JobSubmissionHandler.class, LocalJobSubmissionHandler.class)
.bindConstructor(ExecutorService.class, ExecutorServiceConstructor.class)
- // Bind the message codec for REEF.
- .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
.bindNamedParameter(MaxNumberOfEvaluators.class, MAX_NUMBER_OF_EVALUATORS)
.bindNamedParameter(RootFolder.class, RUNTIME_ROOT_FOLDER)
.bindNamedParameter(JVMHeapSlack.class, JVM_HEAP_SLACK)
+ .bindSetEntry(DriverConfigurationProviders.class, DRIVER_CONFIGURATION_PROVIDERS)
.bindImplementation(RuntimeClasspathProvider.class, LocalClasspathProvider.class)
.build();
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/client/MesosClientConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/client/MesosClientConfiguration.java b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/client/MesosClientConfiguration.java
index 280f2a6..5e5e4e6 100644
--- a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/client/MesosClientConfiguration.java
+++ b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/client/MesosClientConfiguration.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,13 +21,9 @@ package org.apache.reef.runtime.mesos.client;
import org.apache.hadoop.conf.Configuration;
import org.apache.reef.annotations.audience.ClientSide;
import org.apache.reef.annotations.audience.Public;
-import org.apache.reef.client.REEF;
-import org.apache.reef.client.RunningJob;
-import org.apache.reef.runtime.common.client.REEFImplementation;
-import org.apache.reef.runtime.common.client.RunningJobImpl;
+import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
-import org.apache.reef.runtime.common.launch.REEFMessageCodec;
import org.apache.reef.runtime.mesos.MesosClasspathProvider;
import org.apache.reef.runtime.mesos.client.parameters.MasterIp;
import org.apache.reef.runtime.mesos.client.parameters.RootFolder;
@@ -36,7 +32,6 @@ import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
import org.apache.reef.tang.formats.OptionalParameter;
import org.apache.reef.tang.formats.RequiredParameter;
-import org.apache.reef.wake.remote.RemoteConfiguration;
/**
* A ConfigurationModule for the Mesos resource manager
@@ -56,9 +51,7 @@ public class MesosClientConfiguration extends ConfigurationModuleBuilder {
public static final RequiredParameter<String> MASTER_IP = new RequiredParameter<>();
public static final ConfigurationModule CONF = new MesosClientConfiguration()
- .bindImplementation(REEF.class, REEFImplementation.class)
- .bindImplementation(RunningJob.class, RunningJobImpl.class)
- .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .merge(CommonRuntimeConfiguration.CONF)
.bindImplementation(JobSubmissionHandler.class, MesosJobSubmissionHandler.class)
.bindNamedParameter(RootFolder.class, ROOT_FOLDER)
.bindNamedParameter(MasterIp.class, MASTER_IP)
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/774a5d75/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnClientConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnClientConfiguration.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnClientConfiguration.java
index c893dde..c5a7e23 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnClientConfiguration.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/client/YarnClientConfiguration.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,13 +20,11 @@ package org.apache.reef.runtime.yarn.client;
import org.apache.reef.annotations.audience.ClientSide;
import org.apache.reef.annotations.audience.Public;
-import org.apache.reef.client.REEF;
-import org.apache.reef.client.RunningJob;
-import org.apache.reef.runtime.common.client.REEFImplementation;
-import org.apache.reef.runtime.common.client.RunningJobImpl;
+import org.apache.reef.client.parameters.DriverConfigurationProviders;
+import org.apache.reef.common.ConfigurationProvider;
+import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
-import org.apache.reef.runtime.common.launch.REEFMessageCodec;
import org.apache.reef.runtime.common.parameters.JVMHeapSlack;
import org.apache.reef.runtime.yarn.YarnClasspathProvider;
import org.apache.reef.runtime.yarn.client.parameters.JobPriority;
@@ -34,9 +32,9 @@ import org.apache.reef.runtime.yarn.client.parameters.JobQueue;
import org.apache.reef.runtime.yarn.util.YarnConfigurationConstructor;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.tang.formats.OptionalImpl;
import org.apache.reef.tang.formats.OptionalParameter;
import org.apache.reef.util.logging.LoggingSetup;
-import org.apache.reef.wake.remote.RemoteConfiguration;
/**
* A ConfigurationModule for the YARN resourcemanager.
@@ -53,12 +51,13 @@ public class YarnClientConfiguration extends ConfigurationModuleBuilder {
public static final OptionalParameter<Double> JVM_HEAP_SLACK = new OptionalParameter<>();
+ /**
+ * Configuration provides whose Configuration will be merged into all Driver Configuration.
+ */
+ public static final OptionalImpl<ConfigurationProvider> DRIVER_CONFIGURATION_PROVIDERS = new OptionalImpl<>();
+
public static final ConfigurationModule CONF = new YarnClientConfiguration()
- // Bind the common resourcemanager
- .bindImplementation(REEF.class, REEFImplementation.class)
- .bindImplementation(RunningJob.class, RunningJobImpl.class)
- // Bind the message codec for REEF.
- .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .merge(CommonRuntimeConfiguration.CONF)
// Bind YARN
.bindImplementation(JobSubmissionHandler.class, YarnJobSubmissionHandler.class)
// Bind the parameters given by the user
@@ -68,6 +67,7 @@ public class YarnClientConfiguration extends ConfigurationModuleBuilder {
.bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
// Bind external constructors. Taken from YarnExternalConstructors.registerClientConstructors
.bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
+ .bindSetEntry(DriverConfigurationProviders.class, DRIVER_CONFIGURATION_PROVIDERS)
.build();
}