You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/01/23 00:46:56 UTC
[23/51] [partial] incubator-reef git commit: [REEF-93] Move java
sources to lang/java
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightDriverConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightDriverConfiguration.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightDriverConfiguration.java
new file mode 100644
index 0000000..f372fa1
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightDriverConfiguration.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.reef.annotations.audience.ClientSide;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.io.TempFileCreator;
+import org.apache.reef.io.WorkingDirectoryTempFileCreator;
+import org.apache.reef.runtime.common.driver.api.AbstractDriverRuntimeConfiguration;
+import org.apache.reef.runtime.common.driver.api.ResourceLaunchHandler;
+import org.apache.reef.runtime.common.driver.api.ResourceReleaseHandler;
+import org.apache.reef.runtime.common.driver.api.ResourceRequestHandler;
+import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
+import org.apache.reef.runtime.common.parameters.JVMHeapSlack;
+import org.apache.reef.runtime.hdinsight.HDInsightClasspathProvider;
+import org.apache.reef.runtime.yarn.driver.*;
+import org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectory;
+import org.apache.reef.runtime.yarn.driver.parameters.YarnHeartbeatPeriod;
+import org.apache.reef.runtime.yarn.util.YarnConfigurationConstructor;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.tang.formats.OptionalParameter;
+import org.apache.reef.tang.formats.RequiredParameter;
+import org.apache.reef.wake.time.Clock;
+
+/**
+ * ConfigurationModule to create a Driver configuration.
+ */
+@Private
+@ClientSide
+public final class HDInsightDriverConfiguration extends ConfigurationModuleBuilder {
+
+ /**
+ * @see org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectory
+ */
+ public static final RequiredParameter<String> JOB_SUBMISSION_DIRECTORY = new RequiredParameter<>();
+ /**
+ * @see org.apache.reef.runtime.yarn.driver.parameters.YarnHeartbeatPeriod.class
+ */
+ public static final OptionalParameter<Integer> YARN_HEARTBEAT_INTERVAL = new OptionalParameter<>();
+
+ /**
+ * @see AbstractDriverRuntimeConfiguration.JobIdentifier.class
+ */
+ public static final RequiredParameter<String> JOB_IDENTIFIER = new RequiredParameter<>();
+
+ /**
+ * @see AbstractDriverRuntimeConfiguration.EvaluatorTimeout
+ */
+ public static final OptionalParameter<Long> EVALUATOR_TIMEOUT = new OptionalParameter<>();
+
+ /**
+ * The fraction of the container memory NOT to use for the Java Heap.
+ */
+ public static final OptionalParameter<Double> JVM_HEAP_SLACK = new OptionalParameter<>();
+
+ public static final ConfigurationModule CONF = new HDInsightDriverConfiguration()
+
+ // Bind the YARN runtime for the resource manager.
+ .bindImplementation(ResourceLaunchHandler.class, YARNResourceLaunchHandler.class)
+ .bindImplementation(ResourceReleaseHandler.class, YARNResourceReleaseHandler.class)
+ .bindImplementation(ResourceRequestHandler.class, YarnResourceRequestHandler.class)
+ .bindConstructor(YarnConfiguration.class, YarnConfigurationConstructor.class)
+ .bindSetEntry(Clock.RuntimeStartHandler.class, YARNRuntimeStartHandler.class)
+ .bindSetEntry(Clock.RuntimeStopHandler.class, YARNRuntimeStopHandler.class)
+ .bindImplementation(TempFileCreator.class, WorkingDirectoryTempFileCreator.class)
+
+ // Bind the YARN Configuration parameters
+ .bindNamedParameter(JobSubmissionDirectory.class, JOB_SUBMISSION_DIRECTORY)
+ .bindNamedParameter(YarnHeartbeatPeriod.class, YARN_HEARTBEAT_INTERVAL)
+
+ // Bind the fields bound in AbstractDriverRuntimeConfiguration
+ .bindNamedParameter(AbstractDriverRuntimeConfiguration.JobIdentifier.class, JOB_IDENTIFIER)
+ .bindNamedParameter(AbstractDriverRuntimeConfiguration.EvaluatorTimeout.class, EVALUATOR_TIMEOUT)
+ .bindNamedParameter(JVMHeapSlack.class, JVM_HEAP_SLACK)
+ .bindImplementation(RuntimeClasspathProvider.class, HDInsightClasspathProvider.class)
+ .build();
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
new file mode 100644
index 0000000..7ac314a
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.reef.annotations.audience.ClientSide;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.proto.ClientRuntimeProtocol;
+import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
+import org.apache.reef.runtime.common.files.ClasspathProvider;
+import org.apache.reef.runtime.common.files.JobJarMaker;
+import org.apache.reef.runtime.common.files.REEFFileNames;
+import org.apache.reef.runtime.common.launch.JavaLaunchCommandBuilder;
+import org.apache.reef.runtime.common.parameters.JVMHeapSlack;
+import org.apache.reef.runtime.hdinsight.client.yarnrest.*;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Configurations;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.tang.formats.ConfigurationSerializer;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Handles job submission to a HDInsight instance.
+ */
+@ClientSide
+@Private
+public final class HDInsightJobSubmissionHandler implements JobSubmissionHandler {
+
+ private static final Logger LOG = Logger.getLogger(HDInsightJobSubmissionHandler.class.getName());
+
+ private final AzureUploader uploader;
+ private final JobJarMaker jobJarMaker;
+ private final HDInsightInstance hdInsightInstance;
+ private final ConfigurationSerializer configurationSerializer;
+ private final REEFFileNames filenames;
+ private final ClasspathProvider classpath;
+ private final double jvmHeapSlack;
+
+ @Inject
+ HDInsightJobSubmissionHandler(final AzureUploader uploader,
+ final JobJarMaker jobJarMaker,
+ final HDInsightInstance hdInsightInstance,
+ final ConfigurationSerializer configurationSerializer,
+ final REEFFileNames filenames,
+ final ClasspathProvider classpath,
+ final @Parameter(JVMHeapSlack.class) double jvmHeapSlack) {
+ this.uploader = uploader;
+ this.jobJarMaker = jobJarMaker;
+ this.hdInsightInstance = hdInsightInstance;
+ this.configurationSerializer = configurationSerializer;
+ this.filenames = filenames;
+ this.classpath = classpath;
+ this.jvmHeapSlack = jvmHeapSlack;
+ }
+
+ @Override
+ public void close() {
+ LOG.log(Level.WARNING, ".close() is inconsequential with the HDInsight runtime");
+ }
+
+ @Override
+ public void onNext(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) {
+
+ try {
+
+ LOG.log(Level.FINE, "Requesting Application ID from HDInsight.");
+ final ApplicationID applicationID = this.hdInsightInstance.getApplicationID();
+
+ LOG.log(Level.INFO, "Submitting application {0} to YARN.", applicationID.getId());
+
+ LOG.log(Level.FINE, "Creating a job folder on Azure.");
+ final String jobFolderURL = this.uploader.createJobFolder(applicationID.getId());
+
+ LOG.log(Level.FINE, "Assembling Configuration for the Driver.");
+ final Configuration driverConfiguration =
+ makeDriverConfiguration(jobSubmissionProto, applicationID.getId(), jobFolderURL);
+
+ LOG.log(Level.FINE, "Making Job JAR.");
+ final File jobSubmissionJarFile =
+ this.jobJarMaker.createJobSubmissionJAR(jobSubmissionProto, driverConfiguration);
+
+ LOG.log(Level.FINE, "Uploading Job JAR to Azure.");
+ final FileResource uploadedFile = this.uploader.uploadFile(jobSubmissionJarFile);
+
+ LOG.log(Level.FINE, "Assembling application submission.");
+ final String command = getCommandString(jobSubmissionProto);
+
+ final ApplicationSubmission applicationSubmission = new ApplicationSubmission()
+ .setApplicationId(applicationID.getId())
+ .setApplicationName(jobSubmissionProto.getIdentifier())
+ .setResource(getResource(jobSubmissionProto))
+ .setContainerInfo(new ContainerInfo()
+ .addFileResource(this.filenames.getREEFFolderName(), uploadedFile)
+ .addCommand(command));
+
+ this.hdInsightInstance.submitApplication(applicationSubmission);
+ LOG.log(Level.INFO, "Submitted application to HDInsight. The application id is: {0}", applicationID.getId());
+
+ } catch (final IOException ex) {
+ LOG.log(Level.SEVERE, "Error submitting HDInsight request", ex);
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * Extracts the resource demands from the jobSubmissionProto.
+ */
+ private final Resource getResource(
+ final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) {
+
+ return new Resource()
+ .setMemory(String.valueOf(jobSubmissionProto.getDriverMemory()))
+ .setvCores("1");
+ }
+
+ /**
+ * Assembles the command to execute the Driver.
+ */
+ private String getCommandString(
+ final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) {
+ return StringUtils.join(getCommandList(jobSubmissionProto), ' ');
+ }
+
+ /**
+ * Assembles the command to execute the Driver in list form.
+ */
+ private List<String> getCommandList(
+ final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) {
+
+ return new JavaLaunchCommandBuilder()
+ .setJavaPath("%JAVA_HOME%/bin/java")
+ .setErrorHandlerRID(jobSubmissionProto.getRemoteId())
+ .setLaunchID(jobSubmissionProto.getIdentifier())
+ .setConfigurationFileName(this.filenames.getDriverConfigurationPath())
+ .setClassPath(this.classpath.getDriverClasspath())
+ .setMemory(jobSubmissionProto.getDriverMemory())
+ .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName())
+ .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName())
+ .build();
+ }
+
+ private Configuration makeDriverConfiguration(
+ final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto,
+ final String applicationId,
+ final String jobFolderURL) throws IOException {
+
+ final Configuration hdinsightDriverConfiguration = HDInsightDriverConfiguration.CONF
+ .set(HDInsightDriverConfiguration.JOB_IDENTIFIER, applicationId)
+ .set(HDInsightDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolderURL)
+ .set(HDInsightDriverConfiguration.JVM_HEAP_SLACK, this.jvmHeapSlack)
+ .build();
+
+ return Configurations.merge(
+ this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration()),
+ hdinsightDriverConfiguration);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
new file mode 100644
index 0000000..22e3d1d
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client;
+
+import org.apache.reef.runtime.hdinsight.parameters.*;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Configurations;
+import org.apache.reef.tang.formats.AvroConfigurationSerializer;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.tang.formats.RequiredParameter;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Configuration module to setup REEF to submit jobs to HDInsight.
+ */
+public final class HDInsightRuntimeConfiguration extends ConfigurationModuleBuilder {
+
+ /**
+ * The URL of the hdinsight web service. E.g. http://services.mycompany.com:1234/templeton/v1/
+ */
+ public static final RequiredParameter<String> URL = new RequiredParameter<>();
+
+ /**
+ * The Storage account to be used by Azure.
+ */
+ public static final RequiredParameter<String> STORAGE_ACCOUNT_NAME = new RequiredParameter<>();
+
+ /**
+ * The Storage account key to be used by Azure.
+ */
+ public static final RequiredParameter<String> STORAGE_ACCOUNT_KEY = new RequiredParameter<>();
+
+ /**
+ * The Container name to be used by Azure.
+ */
+ public static final RequiredParameter<String> CONTAINER_NAME = new RequiredParameter<>();
+
+ /**
+ * The username to be used for connecting to hdinsight.
+ */
+ public static final RequiredParameter<String> USER_NAME = new RequiredParameter<>();
+
+ /**
+ * The password to be used for connecting to hdinsight.
+ */
+ public static final RequiredParameter<String> PASSWORD = new RequiredParameter<>();
+
+ /**
+ * The environment variable that holds the path to the default configuration file
+ */
+ public static final String HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE = "REEF_HDI_CONF";
+
+ public static final ConfigurationModule CONF = new HDInsightRuntimeConfiguration()
+ .merge(HDInsightRuntimeConfigurationStatic.CONF)
+ .bindNamedParameter(AzureStorageAccountName.class, STORAGE_ACCOUNT_NAME)
+ .bindNamedParameter(AzureStorageAccountKey.class, STORAGE_ACCOUNT_KEY)
+ .bindNamedParameter(AzureStorageAccountContainerName.class, CONTAINER_NAME)
+ .bindNamedParameter(HDInsightInstanceURL.class, URL)
+ .bindNamedParameter(HDInsightUsername.class, USER_NAME)
+ .bindNamedParameter(HDInsightPassword.class, PASSWORD)
+ .build();
+
+ /**
+ * Returns a HDInsight runtime configuration from the credentials stored in the given file.
+ *
+ * @param file
+ * @return a HDInsight runtime configuration from the credentials stored in the given file.
+ * @throws IOException if the file can't be read
+ */
+ public static Configuration fromTextFile(final File file) throws IOException {
+ final Configuration loaded = new AvroConfigurationSerializer().fromTextFile(file);
+ final Configuration staticConfiguration = HDInsightRuntimeConfigurationStatic.CONF.build();
+ return Configurations.merge(loaded, staticConfiguration);
+ }
+
+ /**
+ * @return the RuntimeConfiguration that is stored in a file refered to
+ * by the environment variable HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE.
+ * @throws IOException
+ * @see HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE
+ */
+ public static Configuration fromEnvironment() throws IOException {
+
+ final String configurationPath =
+ System.getenv(HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE);
+
+ if (null == configurationPath) {
+ throw new IOException("Environment Variable " +
+ HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE +
+ " not set.");
+ }
+
+ final File configurationFile = new File(configurationPath);
+ if (!configurationFile.canRead()) {
+ throw new IOException("Environment Variable " +
+ HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE +
+ " points to a file " + configurationFile.getAbsolutePath() +
+ " which can't be read."
+ );
+ }
+
+ return fromTextFile(configurationFile);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
new file mode 100644
index 0000000..e84e77f
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfigurationStatic.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.reef.client.REEF;
+import org.apache.reef.client.RunningJob;
+import org.apache.reef.runtime.common.client.REEFImplementation;
+import org.apache.reef.runtime.common.client.RunningJobImpl;
+import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
+import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
+import org.apache.reef.runtime.common.launch.REEFMessageCodec;
+import org.apache.reef.runtime.hdinsight.HDInsightClasspathProvider;
+import org.apache.reef.runtime.hdinsight.client.sslhacks.DefaultClientConstructor;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.util.logging.LoggingSetup;
+import org.apache.reef.wake.remote.RemoteConfiguration;
+
+/**
+ * The static part of the HDInsightRuntimeConfiguration.
+ */
+public final class HDInsightRuntimeConfigurationStatic extends ConfigurationModuleBuilder {
+ static {
+ LoggingSetup.setupCommonsLogging();
+ }
+
+ public static final ConfigurationModule CONF = new HDInsightRuntimeConfigurationStatic()
+ .bindImplementation(REEF.class, REEFImplementation.class)
+ .bindImplementation(RunningJob.class, RunningJobImpl.class)
+ .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .bindImplementation(JobSubmissionHandler.class, HDInsightJobSubmissionHandler.class)
+ .bindConstructor(CloseableHttpClient.class, DefaultClientConstructor.class)
+ .bindImplementation(RuntimeClasspathProvider.class, HDInsightClasspathProvider.class)
+ .build();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
new file mode 100644
index 0000000..a96f5e5
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client;
+
+import org.apache.reef.runtime.hdinsight.parameters.*;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Configurations;
+import org.apache.reef.tang.formats.AvroConfigurationSerializer;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.tang.formats.RequiredParameter;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Same as HDInsightRuntimeConfiguration, but ignores SSL errors on submission.
+ */
+public final class UnsafeHDInsightRuntimeConfiguration extends ConfigurationModuleBuilder {
+
+ /**
+ * The URL of the hdinsight web service. E.g. http://services.mycompany.com:1234/templeton/v1/
+ */
+ public static final RequiredParameter<String> URL = new RequiredParameter<>();
+
+ /**
+ * The Storage account to be used by Azure.
+ */
+ public static final RequiredParameter<String> STORAGE_ACCOUNT_NAME = new RequiredParameter<>();
+
+ /**
+ * The Storage account key to be used by Azure.
+ */
+ public static final RequiredParameter<String> STORAGE_ACCOUNT_KEY = new RequiredParameter<>();
+
+ /**
+ * The Container name to be used by Azure.
+ */
+ public static final RequiredParameter<String> CONTAINER_NAME = new RequiredParameter<>();
+
+ /**
+ * The username to be used for connecting to hdinsight.
+ */
+ public static final RequiredParameter<String> USER_NAME = new RequiredParameter<>();
+
+ /**
+ * The password to be used for connecting to hdinsight.
+ */
+ public static final RequiredParameter<String> PASSWORD = new RequiredParameter<>();
+
+ public static final ConfigurationModule CONF = new UnsafeHDInsightRuntimeConfiguration()
+ .merge(UnsafeHDInsightRuntimeConfigurationStatic.CONF)
+ .bindNamedParameter(AzureStorageAccountName.class, STORAGE_ACCOUNT_NAME)
+ .bindNamedParameter(AzureStorageAccountKey.class, STORAGE_ACCOUNT_KEY)
+ .bindNamedParameter(AzureStorageAccountContainerName.class, CONTAINER_NAME)
+ .bindNamedParameter(HDInsightInstanceURL.class, URL)
+ .bindNamedParameter(HDInsightUsername.class, USER_NAME)
+ .bindNamedParameter(HDInsightPassword.class, PASSWORD)
+ .build();
+
+ /**
+ * Returns an UNSAFE HDInsight runtime configuration from the credentials stored in the given file.
+ *
+ * @param file
+ * @return an UNSAFE HDInsight runtime configuration from the credentials stored in the given file.
+ * @throws java.io.IOException if the file can't be read
+ */
+ public static Configuration fromTextFile(final File file) throws IOException {
+ final Configuration loaded = new AvroConfigurationSerializer().fromTextFile(file);
+ final Configuration staticConfiguration = UnsafeHDInsightRuntimeConfigurationStatic.CONF.build();
+ return Configurations.merge(loaded, staticConfiguration);
+ }
+
+ /**
+ * @return the RuntimeConfiguration that is stored in a file refered to by the environment
+ * variable HDInsightRuntimeConfiguration.HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE.
+ * @throws IOException
+ * @see HDInsightRuntimeConfiguration.HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE
+ */
+ public static Configuration fromEnvironment() throws IOException {
+
+ final String configurationPath = System.getenv(
+ HDInsightRuntimeConfiguration.HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE);
+
+ if (null == configurationPath) {
+ throw new IOException("Environment Variable " +
+ HDInsightRuntimeConfiguration.HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE +
+ " not set.");
+ }
+
+ final File configurationFile = new File(configurationPath);
+ if (!configurationFile.canRead()) {
+ throw new IOException("Environment Variable " +
+ HDInsightRuntimeConfiguration.HDINSIGHT_CONFIGURATION_FILE_ENVIRONMENT_VARIABLE +
+ " points to a file " + configurationFile.getAbsolutePath() +
+ " which can't be read."
+ );
+ }
+
+ return fromTextFile(configurationFile);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
new file mode 100644
index 0000000..37a46bf
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.reef.client.REEF;
+import org.apache.reef.client.RunningJob;
+import org.apache.reef.runtime.common.client.REEFImplementation;
+import org.apache.reef.runtime.common.client.RunningJobImpl;
+import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
+import org.apache.reef.runtime.common.launch.REEFMessageCodec;
+import org.apache.reef.runtime.hdinsight.client.sslhacks.UnsafeClientConstructor;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.util.logging.LoggingSetup;
+import org.apache.reef.wake.remote.RemoteConfiguration;
+
+/**
+ * The static part of the UnsafeHDInsightRuntimeConfiguration
+ */
+public final class UnsafeHDInsightRuntimeConfigurationStatic extends ConfigurationModuleBuilder {
+ static {
+ LoggingSetup.setupCommonsLogging();
+ }
+
+ public static final ConfigurationModule CONF = new UnsafeHDInsightRuntimeConfigurationStatic()
+ .bindImplementation(REEF.class, REEFImplementation.class)
+ .bindImplementation(RunningJob.class, RunningJobImpl.class)
+ .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .bindImplementation(JobSubmissionHandler.class, HDInsightJobSubmissionHandler.class)
+ .bindConstructor(CloseableHttpClient.class, UnsafeClientConstructor.class)
+ .build();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/package-info.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/package-info.java
new file mode 100644
index 0000000..042207e
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * The client to submit jobs to HDInsight.
+ */
+package org.apache.reef.runtime.hdinsight.client;
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/DefaultClientConstructor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/DefaultClientConstructor.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/DefaultClientConstructor.java
new file mode 100644
index 0000000..47ce1d5
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/DefaultClientConstructor.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.sslhacks;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.reef.tang.ExternalConstructor;
+
+import javax.inject.Inject;
+
+/**
+ * Default Client constructor with default SSL checks.
+ */
+public final class DefaultClientConstructor implements ExternalConstructor<CloseableHttpClient> {
+ @Inject
+ DefaultClientConstructor() {
+ }
+
+ @Override
+ public CloseableHttpClient newInstance() {
+ return HttpClients.createDefault();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeClientConstructor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeClientConstructor.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeClientConstructor.java
new file mode 100644
index 0000000..a696540
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeClientConstructor.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.sslhacks;
+
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.conn.BasicClientConnectionManager;
+import org.apache.reef.tang.ExternalConstructor;
+
+import javax.inject.Inject;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A Client constructor that produces Clients that do not check SSL.
+ */
+public final class UnsafeClientConstructor implements ExternalConstructor<CloseableHttpClient> {
+
+ @Inject
+ UnsafeClientConstructor() {
+ Logger.getLogger(UnsafeClientConstructor.class.getName())
+ .log(Level.SEVERE, "DANGER: INSTANTIATING HTTP CLIENT WITH NO SSL CHECKS.");
+ }
+
+ @Override
+ public CloseableHttpClient newInstance() {
+ try {
+ final SSLSocketFactory socketFactory = new SSLSocketFactory(this.getSSLContext());
+ socketFactory.setHostnameVerifier(new UnsafeHostNameVerifier());
+ final SchemeRegistry schemeRegistry = new SchemeRegistry();
+ schemeRegistry.register(new Scheme("https", 443, socketFactory));
+ final ClientConnectionManager clientConnectionManager = new BasicClientConnectionManager(schemeRegistry);
+ return new DefaultHttpClient(clientConnectionManager);
+ } catch (final KeyManagementException | NoSuchAlgorithmException ex) {
+ throw new RuntimeException("Unable to instantiate HTTP Client", ex);
+ }
+ }
+
+ private SSLContext getSSLContext() throws KeyManagementException, NoSuchAlgorithmException {
+ final SSLContext sc = SSLContext.getInstance("TLS");
+ sc.init(new KeyManager[0], new TrustManager[]{new UnsafeTrustManager()}, new SecureRandom());
+ return sc;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeHostNameVerifier.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeHostNameVerifier.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeHostNameVerifier.java
new file mode 100644
index 0000000..1ff83f3
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeHostNameVerifier.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.sslhacks;
+
+import org.apache.http.conn.ssl.X509HostnameVerifier;
+
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocket;
+import java.io.IOException;
+import java.security.cert.X509Certificate;
+
+final class UnsafeHostNameVerifier implements X509HostnameVerifier {
+
+ @Override
+ public void verify(String host, SSLSocket ssl) throws IOException {
+
+ }
+
+ @Override
+ public void verify(String host, X509Certificate cert) throws SSLException {
+
+ }
+
+ @Override
+ public void verify(String host, String[] cns, String[] subjectAlts) throws SSLException {
+
+ }
+
+ @Override
+ public boolean verify(String s, SSLSession sslSession) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeTrustManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeTrustManager.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeTrustManager.java
new file mode 100644
index 0000000..3e6e1e6
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/sslhacks/UnsafeTrustManager.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.sslhacks;
+
+import javax.inject.Inject;
+import javax.net.ssl.X509TrustManager;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+/**
+ * A TrustManager that trusts all certificates. Basically the "GOTO FAIL" bug implemented in Java.
+ * <p/>
+ * Hence: DO NOT USE THIS CLASS UNLESS DEBUGGING.
+ */
+final class UnsafeTrustManager implements X509TrustManager {
+ @Inject
+ UnsafeTrustManager() {
+ }
+
+ @Override
+ public void checkClientTrusted(final X509Certificate[] x509Certificates, final String s) throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(final X509Certificate[] x509Certificates, final String s) throws CertificateException {
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[0];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
new file mode 100644
index 0000000..d3ead6b
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+/**
+ * Represents the response to an application ID request.
+ */
+public final class ApplicationID {
+
+ private String id;
+ private Resource resource;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(final Resource resource) {
+ this.resource = resource;
+ }
+
+ @Override
+ public String toString() {
+ return "ApplicationID{" +
+ "id='" + id + '\'' +
+ ", resource=" + resource +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
new file mode 100644
index 0000000..8586e5e
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by marku_000 on 2014-06-30.
+ */
+public class ApplicationResponse {
+
+ private Map<String, List<ApplicationState>> apps;
+
+ public Map<String, List<ApplicationState>> getApps() {
+ return apps;
+ }
+
+ public void setApps(Map<String, List<ApplicationState>> apps) {
+ this.apps = apps;
+ }
+
+ public List<ApplicationState> getApplicationStates() {
+ return apps.get("app");
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
new file mode 100644
index 0000000..6825dd9
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+/**
+ * Created by marku_000 on 2014-06-30.
+ */
+public class ApplicationState {
+ private String progress;
+ private String queue;
+ private String trackingUI;
+ private String state;
+ private String amContainerLogs;
+ private String applicationType;
+ private int runningContainers;
+ private int allocatedMB;
+ private long elapsedTime;
+ private String amHostHttpAddress;
+ private String id;
+ private String finalStatus;
+ private String trackingUrl;
+ private int allocatedVCores;
+ private long finishedTime;
+ private String applicationTags;
+ private String name;
+ private long clusterId;
+ private String user;
+ private String diagnostics;
+ private long startedTime;
+
+ public String getProgress() {
+ return progress;
+ }
+
+ public void setProgress(String progress) {
+ this.progress = progress;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ public String getTrackingUI() {
+ return trackingUI;
+ }
+
+ public void setTrackingUI(String trackingUI) {
+ this.trackingUI = trackingUI;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ public String getAmContainerLogs() {
+ return amContainerLogs;
+ }
+
+ public void setAmContainerLogs(String amContainerLogs) {
+ this.amContainerLogs = amContainerLogs;
+ }
+
+ public String getApplicationType() {
+ return applicationType;
+ }
+
+ public void setApplicationType(String applicationType) {
+ this.applicationType = applicationType;
+ }
+
+ public int getRunningContainers() {
+ return runningContainers;
+ }
+
+ public void setRunningContainers(int runningContainers) {
+ this.runningContainers = runningContainers;
+ }
+
+ public int getAllocatedMB() {
+ return allocatedMB;
+ }
+
+ public void setAllocatedMB(int allocatedMB) {
+ this.allocatedMB = allocatedMB;
+ }
+
+ public long getElapsedTime() {
+ return elapsedTime;
+ }
+
+ public void setElapsedTime(long elapsedTime) {
+ this.elapsedTime = elapsedTime;
+ }
+
+ public String getAmHostHttpAddress() {
+ return amHostHttpAddress;
+ }
+
+ public void setAmHostHttpAddress(String amHostHttpAddress) {
+ this.amHostHttpAddress = amHostHttpAddress;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getFinalStatus() {
+ return finalStatus;
+ }
+
+ public void setFinalStatus(String finalStatus) {
+ this.finalStatus = finalStatus;
+ }
+
+ public String getTrackingUrl() {
+ return trackingUrl;
+ }
+
+ public void setTrackingUrl(String trackingUrl) {
+ this.trackingUrl = trackingUrl;
+ }
+
+ public int getAllocatedVCores() {
+ return allocatedVCores;
+ }
+
+ public void setAllocatedVCores(int allocatedVCores) {
+ this.allocatedVCores = allocatedVCores;
+ }
+
+ public long getFinishedTime() {
+ return finishedTime;
+ }
+
+ public void setFinishedTime(long finishedTime) {
+ this.finishedTime = finishedTime;
+ }
+
+ public String getApplicationTags() {
+ return applicationTags;
+ }
+
+ public void setApplicationTags(String applicationTags) {
+ this.applicationTags = applicationTags;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public long getClusterId() {
+ return clusterId;
+ }
+
+ public void setClusterId(long clusterId) {
+ this.clusterId = clusterId;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ public long getStartedTime() {
+ return startedTime;
+ }
+
+ public void setStartedTime(long startedTime) {
+ this.startedTime = startedTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
new file mode 100644
index 0000000..77455e4
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+/**
+ * Represents an ApplicationSubmission to the YARN REST API.
+ */
+public final class ApplicationSubmission {
+
+ public static final String DEFAULT_QUEUE = "default";
+ private String queue = DEFAULT_QUEUE;
+
+ public static final String DEFAULT_PRIORITY = "3";
+ private String priority = DEFAULT_PRIORITY;
+
+ public static final String DEFAULT_MAX_ATTEMPTS = "1";
+ private String maxAppAttempts = DEFAULT_MAX_ATTEMPTS;
+
+ public static final String DEFAULT_APPLICATION_TYPE = "YARN";
+ private String applicationType = DEFAULT_APPLICATION_TYPE;
+
+ public static final String DEFAULT_KEEP_CONTAINERS = "false";
+ private String keepContainers = DEFAULT_KEEP_CONTAINERS;
+
+ public static final String DEFAULT_IS_UNMANAGED_AM = "false";
+ private String isUnmanagedAM = DEFAULT_IS_UNMANAGED_AM;
+
+ public static final String DEFAULT_CANCEL_TOKENS_WHEN_COMPLETE = "true";
+ private String cancelTokensWhenComplete = DEFAULT_CANCEL_TOKENS_WHEN_COMPLETE;
+
+ private String applicationId;
+ private String applicationName;
+ private ContainerInfo containerInfo;
+ private Resource resource;
+
+ public String getApplicationId() {
+ return applicationId;
+ }
+
+ public ApplicationSubmission setApplicationId(String applicationId) {
+ this.applicationId = applicationId;
+ return this;
+ }
+
+ public String getApplicationName() {
+ return applicationName;
+ }
+
+ public ApplicationSubmission setApplicationName(String applicationName) {
+ this.applicationName = applicationName;
+ return this;
+ }
+
+ public String getApplicationType() {
+ return applicationType;
+ }
+
+ public ApplicationSubmission setApplicationType(String applicationType) {
+ this.applicationType = applicationType;
+ return this;
+ }
+
+ public String isCancelTokensWhenComplete() {
+ return cancelTokensWhenComplete;
+ }
+
+ public ApplicationSubmission setCancelTokensWhenComplete(String cancelTokensWhenComplete) {
+ this.cancelTokensWhenComplete = cancelTokensWhenComplete;
+ return this;
+ }
+
+ public ContainerInfo getContainerInfo() {
+ return containerInfo;
+ }
+
+ public ApplicationSubmission setContainerInfo(ContainerInfo containerInfo) {
+ this.containerInfo = containerInfo;
+ return this;
+ }
+
+ public String isUnmanagedAM() {
+ return isUnmanagedAM;
+ }
+
+ public ApplicationSubmission setUnmanagedAM(String isUnmanagedAM) {
+ this.isUnmanagedAM = isUnmanagedAM;
+ return this;
+ }
+
+ public String isKeepContainers() {
+ return keepContainers;
+ }
+
+ public ApplicationSubmission setKeepContainers(String keepContainers) {
+ this.keepContainers = keepContainers;
+ return this;
+ }
+
+ public String getMaxAppAttempts() {
+ return maxAppAttempts;
+ }
+
+ public ApplicationSubmission setMaxAppAttempts(String maxAppAttempts) {
+ this.maxAppAttempts = maxAppAttempts;
+ return this;
+ }
+
+ public String getPriority() {
+ return priority;
+ }
+
+ public ApplicationSubmission setPriority(String priority) {
+ this.priority = priority;
+ return this;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public ApplicationSubmission setQueue(String queue) {
+ this.queue = queue;
+ return this;
+ }
+
+ public Resource getResource() {
+ return resource;
+ }
+
+ public ApplicationSubmission setResource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "ApplicationSubmission{" +
+ "queue='" + queue + '\'' +
+ ", priority=" + priority +
+ ", maxAppAttempts=" + maxAppAttempts +
+ ", applicationType='" + applicationType + '\'' +
+ ", keepContainers=" + keepContainers +
+ ", applicationId='" + applicationId + '\'' +
+ ", applicationName='" + applicationName + '\'' +
+ ", containerInfo=" + containerInfo +
+ ", isUnmanagedAM=" + isUnmanagedAM +
+ ", cancelTokensWhenComplete=" + cancelTokensWhenComplete +
+ ", resource=" + resource +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java
new file mode 100644
index 0000000..3c779d5
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents a ContainerInfo in the YARN REST APIs.
+ */
+public final class ContainerInfo {
+
+ public static final String DEFAULT_SERVICE_DATA = null;
+ private String serviceData = DEFAULT_SERVICE_DATA;
+
+ public static final String DEFAULT_TOKENS = "";
+ private String tokens = DEFAULT_TOKENS;
+
+ public static final String DEFAULT_ACLS = null;
+ private String acls = DEFAULT_ACLS;
+
+ private List<String> commands = new ArrayList<>();
+ private Map<String, EnvironmentEntry> environment = new HashMap<>();
+ private Map<String, LocalResourcesEntry> localResources = new HashMap<>();
+
+ /**
+ * Adds an environment variable.
+ *
+ * @param key the name of the variable
+ * @param value the value it shall take
+ * @return this
+ */
+ public ContainerInfo addEnvironment(final String key, final String value) {
+ this.environment.put("entry", new EnvironmentEntry(key, value));
+ return this;
+ }
+
+ /**
+ * Adds a command to the command list to be executed
+ *
+ * @param command
+ * @return this
+ */
+ public ContainerInfo addCommand(final String command) {
+ this.commands.add(command);
+ return this;
+ }
+
+ public ContainerInfo addFileResource(final String key, final FileResource fileResource) {
+ this.localResources.put("entry", new LocalResourcesEntry(key, fileResource));
+ return this;
+ }
+
+ public String getServiceData() {
+ return this.serviceData;
+ }
+
+ public ContainerInfo setServiceData(final String serviceData) {
+ this.serviceData = serviceData;
+ return this;
+ }
+
+ public String getTokens() {
+ return this.tokens;
+ }
+
+ public ContainerInfo setTokens(final String tokens) {
+ this.tokens = tokens;
+ return this;
+ }
+
+ public String getAcls() {
+ return this.acls;
+ }
+
+ public ContainerInfo setAcls(final String acls) {
+ this.acls = acls;
+ return this;
+ }
+
+ public Map<String, EnvironmentEntry> getEnvironment() {
+ return this.environment;
+ }
+
+ public void setEnvironment(final Map<String, EnvironmentEntry> environment) {
+ this.environment = environment;
+ }
+
+ public List<String> getCommands() {
+ return this.commands;
+ }
+
+ public ContainerInfo setCommands(final List<String> commands) {
+ this.commands = commands;
+ return this;
+ }
+
+ public Map<String, LocalResourcesEntry> getLocalResources() {
+ return this.localResources;
+ }
+
+ public ContainerInfo setLocalResources(final Map<String, LocalResourcesEntry> localResources) {
+ this.localResources = localResources;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java
new file mode 100644
index 0000000..8444d47
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+/**
+ * An Entry in the Environment field of an ApplicationSubmission
+ */
+public final class EnvironmentEntry {
+
+ private String key;
+ private String value;
+
+ public EnvironmentEntry(final String key, final String value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ public String getKey() {
+ return this.key;
+ }
+
+ public void setKey(final String key) {
+ this.key = key;
+ }
+
+ public String getValue() {
+ return this.value;
+ }
+
+ public void setValue(final String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return "EnvironmentEntry{" +
+ "key='" + this.key + '\'' +
+ ", value='" + this.value + '\'' +
+ '}';
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final EnvironmentEntry that = (EnvironmentEntry) o;
+
+ return (this.key == that.key || (this.key != null && this.key.equals(that.key)))
+ && (this.value == that.value || (this.value != null && this.value.equals(that.value)));
+ }
+
+ @Override
+ public int hashCode() {
+ int result = this.key != null ? this.key.hashCode() : 0;
+ result = 31 * result + (this.value != null ? this.value.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java
new file mode 100644
index 0000000..519228c
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+public final class FileResource {
+
+ public static final String TYPE_FILE = "FILE";
+ public static final String TYPE_ARCHIVE = "ARCHIVE";
+
+ public static final String VISIBILITY_APPLICATION = "APPLICATION";
+
+ private String url;
+ private String type;
+ private String visibility;
+ private String size;
+ private String timestamp;
+
+ public String getUrl() {
+ return this.url;
+ }
+
+ public FileResource setUrl(final String url) {
+ this.url = url;
+ return this;
+ }
+
+ public String getType() {
+ return this.type;
+ }
+
+ public FileResource setType(final String type) {
+ this.type = type;
+ return this;
+ }
+
+ public String getVisibility() {
+ return this.visibility;
+ }
+
+ public FileResource setVisibility(final String visibility) {
+ this.visibility = visibility;
+ return this;
+ }
+
+ public String getSize() {
+ return this.size;
+ }
+
+ public FileResource setSize(final String size) {
+ this.size = size;
+ return this;
+ }
+
+ public String getTimestamp() {
+ return this.timestamp;
+ }
+
+ public FileResource setTimestamp(final String timestamp) {
+ this.timestamp = timestamp;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "FileResource{" +
+ "url='" + url + '\'' +
+ ", type='" + type + '\'' +
+ ", visibility='" + visibility + '\'' +
+ ", size=" + size +
+ ", timestamp=" + timestamp +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
new file mode 100644
index 0000000..2bbad8a
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.apache.reef.runtime.hdinsight.parameters.HDInsightInstanceURL;
+import org.apache.reef.runtime.hdinsight.parameters.HDInsightPassword;
+import org.apache.reef.runtime.hdinsight.parameters.HDInsightUsername;
+import org.apache.reef.tang.annotations.Parameter;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Represents an HDInsight instance.
+ */
+public final class HDInsightInstance {
+
+ private static final Logger LOG = Logger.getLogger(HDInsightInstance.class.getName());
+ private static final String APPLICATION_KILL_MESSAGE = "{\"app:{\"state\":\"KILLED\"}}";
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final Header[] headers;
+ private final HttpClientContext httpClientContext;
+
+ private final String instanceUrl;
+ private final String username;
+ private final CloseableHttpClient httpClient;
+
+ @Inject
+ HDInsightInstance(final @Parameter(HDInsightUsername.class) String username,
+ final @Parameter(HDInsightPassword.class) String password,
+ final @Parameter(HDInsightInstanceURL.class) String instanceUrl,
+ final CloseableHttpClient client) throws URISyntaxException, IOException {
+ this.httpClient = client;
+ this.instanceUrl = instanceUrl.endsWith("/") ? instanceUrl : instanceUrl + "/";
+ this.username = username;
+ final String host = this.getHost();
+ this.headers = new Header[]{
+ new BasicHeader("Host", host)
+ };
+ this.httpClientContext = getClientContext(host, username, password);
+ }
+
+ /**
+ * Request an ApplicationId from the cluster.
+ *
+ * @return
+ * @throws IOException
+ */
+ public ApplicationID getApplicationID() throws IOException {
+ final String url = "ws/v1/cluster/appids?user.name=" + this.username;
+ final HttpPost post = preparePost(url);
+ try (final CloseableHttpResponse response = this.httpClient.execute(post, this.httpClientContext)) {
+ final String message = IOUtils.toString(response.getEntity().getContent());
+ final ApplicationID result = this.objectMapper.readValue(message, ApplicationID.class);
+ return result;
+ }
+ }
+
+ /**
+ * Submits an application for execution.
+ *
+ * @param applicationSubmission
+ * @throws IOException
+ */
+ public void submitApplication(final ApplicationSubmission applicationSubmission) throws IOException {
+
+ final String applicationId = applicationSubmission.getApplicationId();
+ final String url = "ws/v1/cluster/apps/" + applicationId + "?user.name=" + this.username;
+ final HttpPost post = preparePost(url);
+
+ final StringWriter writer = new StringWriter();
+ try {
+ this.objectMapper.writeValue(writer, applicationSubmission);
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ final String message = writer.toString();
+ LOG.log(Level.FINE, "Sending:\n{0}", message.replace("\n", "\n\t"));
+ post.setEntity(new StringEntity(message, ContentType.APPLICATION_JSON));
+
+ try (final CloseableHttpResponse response = this.httpClient.execute(post, this.httpClientContext)) {
+ final String responseMessage = IOUtils.toString(response.getEntity().getContent());
+ LOG.log(Level.FINE, "Response: {0}", responseMessage.replace("\n", "\n\t"));
+ }
+ }
+
+ /**
+ * Issues a YARN kill command to the application.
+ *
+ * @param applicationId
+ */
+ public void killApplication(final String applicationId) {
+ throw new NotImplementedException();
+ }
+
+ public List<ApplicationState> listApplications() throws IOException {
+ final String url = "ws/v1/cluster/apps";
+ final HttpGet get = prepareGet(url);
+ try (final CloseableHttpResponse response = this.httpClient.execute(get, this.httpClientContext)) {
+ final String message = IOUtils.toString(response.getEntity().getContent());
+ final ApplicationResponse result = this.objectMapper.readValue(message, ApplicationResponse.class);
+ return result.getApplicationStates();
+ }
+ }
+
+ /**
+ * @param applicationId
+ * @return the URL that can be used to issue application level messages.
+ */
+ public String getApplicationURL(final String applicationId) {
+ return "ws/v1/cluster/apps/" + applicationId;
+ }
+
+ private final String getHost() throws URISyntaxException {
+ final URI uri = new URI(this.instanceUrl);
+ return uri.getHost();
+ }
+
+ /**
+ * Creates a HttpGet request with all the common headers.
+ *
+ * @param url
+ * @return
+ */
+ private HttpGet prepareGet(final String url) {
+ final HttpGet httpGet = new HttpGet(this.instanceUrl + url);
+ for (final Header header : this.headers) {
+ httpGet.addHeader(header);
+ }
+ return httpGet;
+ }
+
+ /**
+ * Creates a HttpPost request with all the common headers.
+ *
+ * @param url
+ * @return
+ */
+ private HttpPost preparePost(final String url) {
+ final HttpPost httpPost = new HttpPost(this.instanceUrl + url);
+ for (final Header header : this.headers) {
+ httpPost.addHeader(header);
+ }
+ return httpPost;
+ }
+
+
+ private HttpClientContext getClientContext(final String hostname, final String username, final String password) throws IOException {
+ final HttpHost targetHost = new HttpHost(hostname, 443, "https");
+ final HttpClientContext result = HttpClientContext.create();
+ // Setup credentials provider
+ final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+ credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
+ result.setCredentialsProvider(credentialsProvider);
+
+ // Setup preemptive authentication
+ final AuthCache authCache = new BasicAuthCache();
+ final BasicScheme basicAuth = new BasicScheme();
+ authCache.put(targetHost, basicAuth);
+ result.setAuthCache(authCache);
+ final HttpGet httpget = new HttpGet("/");
+
+ // Prime the cache
+ try (final CloseableHttpResponse response = this.httpClient.execute(targetHost, httpget, result)) {
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
new file mode 100644
index 0000000..f73c31e
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+public final class LocalResourcesEntry {
+
+ private String key;
+ private FileResource value;
+
+ public LocalResourcesEntry(final String key, final FileResource value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ public String getKey() {
+ return this.key;
+ }
+
+ public LocalResourcesEntry setKey(final String key) {
+ this.key = key;
+ return this;
+ }
+
+ public FileResource getValue() {
+ return this.value;
+ }
+
+ public LocalResourcesEntry setValue(final FileResource value) {
+ this.value = value;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
new file mode 100644
index 0000000..7e1b647
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+/**
+ * Represents the resoure field in the YARN REST API
+ */
+public final class Resource {
+
+ private String memory;
+ private String vCores;
+
+ public String getMemory() {
+ return this.memory;
+ }
+
+ public Resource setMemory(final String memory) {
+ this.memory = memory;
+ return this;
+ }
+
+ public String getvCores() {
+ return this.vCores;
+ }
+
+ public Resource setvCores(final String vCores) {
+ this.vCores = vCores;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "Resource{" +
+ "memory=" + this.memory +
+ ", vCores=" + this.vCores +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/package-info.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/package-info.java
new file mode 100644
index 0000000..0fa4a76
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Some minimal abstraction over the YARN REST API.
+ *
+ * Note: This will likely get an overhaul as that API gets closer to finalization.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/package-info.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/package-info.java
new file mode 100644
index 0000000..94acb4c
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * HDInsight support for REEF.
+ */
+package org.apache.reef.runtime.hdinsight;
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/parameters/AzureStorageAccountContainerName.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/parameters/AzureStorageAccountContainerName.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/parameters/AzureStorageAccountContainerName.java
new file mode 100644
index 0000000..8f51421
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/parameters/AzureStorageAccountContainerName.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.parameters;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * The Storage account container name to be used by Azure.
+ */
+@NamedParameter(doc = "The Storage account container name to be used by Azure")
+public final class AzureStorageAccountContainerName implements Name<String> {
+}